Golang utf8.RuneCount函数代码示例

func (field_editor *FieldEditor) handleKeyEvent(event termbox.Event) (string, bool) {
	is_done := false
	if event.Key == termbox.KeyEnter {
		is_done = true
	} else if event.Key == termbox.KeyEsc {
		is_done = true
		field_editor.value = nil
	} else if event.Key == termbox.KeyArrowLeft {
		if field_editor.cursor_pos > 0 {
			field_editor.cursor_pos--
		}
	} else if event.Key == termbox.KeyArrowUp || event.Key == termbox.KeyCtrlA {
		field_editor.cursor_pos = 0
	} else if event.Key == termbox.KeyArrowRight {
		if field_editor.cursor_pos < utf8.RuneCount(field_editor.value) {
			field_editor.cursor_pos++
		}
	} else if event.Key == termbox.KeyArrowDown || event.Key == termbox.KeyCtrlE {
		field_editor.cursor_pos = utf8.RuneCount(field_editor.value)
	} else if event.Key == termbox.KeyCtrlH || event.Key == termbox.KeyBackspace {
		if field_editor.cursor_pos > 0 {
			field_editor.value = removeRuneAtIndex(field_editor.value, field_editor.cursor_pos-1)
			field_editor.cursor_pos--
		}
	} else if unicode.IsPrint(event.Ch) {
		field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, event.Ch)
		field_editor.cursor_pos++
	} else if event.Key == termbox.KeySpace {
		field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, ' ')
		field_editor.cursor_pos++
	}
	return string(field_editor.value), is_done
}

// parseStringToken takes a token of either LITERAL or STRING type and
// returns the interpreted string, after processing any relevant
// escape sequences.
func (p *parser) parseStringToken(tok *scanner.Token) (string, error) {
	var backslashes bool
	switch tok.Type {
	case scanner.LITERAL:
		backslashes = false
	case scanner.STRING:
		backslashes = true
	default:
		panic("unsupported string token type")
	}

	raw := []byte(tok.Content)
	buf := make([]byte, 0, len(raw))

	for i := 0; i < len(raw); i++ {
		b := raw[i]
		more := len(raw) > (i + 1)

		if b == '$' {
			if more && raw[i+1] == '$' {
				// skip over the second dollar sign
				i++
			}
		} else if backslashes && b == '\\' {
			if !more {
				return "", Errorf(
					ast.Pos{
						Column: tok.Pos.Column + utf8.RuneCount(raw[:i]),
						Line:   tok.Pos.Line,
					},
					`unfinished backslash escape sequence`,
				)
			}
			escapeType := raw[i+1]
			switch escapeType {
			case '\\':
				// skip over the second slash
				i++
			case 'n':
				b = '\n'
				i++
			case '"':
				b = '"'
				i++
			default:
				return "", Errorf(
					ast.Pos{
						Column: tok.Pos.Column + utf8.RuneCount(raw[:i]),
						Line:   tok.Pos.Line,
					},
					`invalid backslash escape sequence`,
				)
			}
		}

		buf = append(buf, b)
	}

	return string(buf), nil
}

func (v *RuneCountValidator) Validate(value string, criteria *Criteria) (bool, error) {
	if criteria == nil {
		return false, errors.New("Criteria for 'rune_count' not enough")
	}
	if criteria.Has("eq") {
		eq, err := criteria.Int("eq")
		if err != nil {
			return false, err
		}
		return utf8.RuneCount([]byte(value)) == eq, nil
	} else if criteria.Has("to") && criteria.Has("from") {
		to, err := criteria.Int("to")
		if err != nil {
			return false, err
		}
		from, err := criteria.Int("from")
		if err != nil {
			return false, err
		}
		count := utf8.RuneCount([]byte(value))
		return count >= from && count <= to, nil
	} else {
		return false, errors.New("Criteria for 'rune_count' not enough")
	}
}

func Timeline(screen_name string, since_id int64) []Post {
	var posts = []Post{}
	for _, p := range sina.TimeLine(0, screen_name, since_id, 20) {
		id := p.Id
		text := p.Text
		link := ""
		if p.Original_Pic != "" {
			link = " ✈ " + p.Original_Pic
		}
		if p.Retweeted_Status != nil {
			if p.Retweeted_Status.User != nil {
				re_user := p.Retweeted_Status.User
				text = text + " //RT @" + re_user.Name + ": " + p.Retweeted_Status.Text
			}

			if p.Retweeted_Status.Original_Pic != "" {
				link = " ✈ " + p.Retweeted_Status.Original_Pic
			}
		}
		len1 := utf8.RuneCount([]byte(text))
		len2 := utf8.RuneCount([]byte(link))
		if len1+len2 > 140 {
			link2 := fmt.Sprintf("http://weibo.com/%d/%s", p.User.Id, sina.QueryMid(id, 1))
			text = SubStringByChar(text, 140-38-len2) + link + " " + link2
		} else {
			text = text + link
		}
		posts = append(posts, Post{id, text})
	}
	return posts
}

func (v *jsonSchema) validateString(currentSchema *jsonSchema, value interface{}, result *ValidationResult, context *jsonContext) {

	// Ignore non strings
	if !isKind(value, reflect.String) {
		return
	}

	stringValue := value.(string)

	// minLength & maxLength:
	if currentSchema.minLength != nil {
		if utf8.RuneCount([]byte(stringValue)) < *currentSchema.minLength {
			result.addError(context, value, fmt.Sprintf(ERROR_MESSAGE_STRING_LENGTH_MUST_BE_GREATER_OR_EQUAL, *currentSchema.minLength))
		}
	}
	if currentSchema.maxLength != nil {
		if utf8.RuneCount([]byte(stringValue)) > *currentSchema.maxLength {
			result.addError(context, value, fmt.Sprintf(ERROR_MESSAGE_STRING_LENGTH_MUST_BE_LOWER_OR_EQUAL, *currentSchema.maxLength))
		}
	}

	// pattern:
	if currentSchema.pattern != nil {
		if !currentSchema.pattern.MatchString(stringValue) {
			result.addError(context, value, fmt.Sprintf(ERROR_MESSAGE_DOES_NOT_MATCH_PATTERN, currentSchema.pattern))

		}
	}

	result.incrementScore()
}

func (v *subSchema) validateString(currentSubSchema *subSchema, value interface{}, result *Result, context *jsonContext) {

	internalLog("validateString %s", context.String())
	internalLog(" %v", value)

	// Ignore non strings
	if !isKind(value, reflect.String) {
		return
	}

	stringValue := value.(string)

	// minLength & maxLength:
	if currentSubSchema.minLength != nil {
		if utf8.RuneCount([]byte(stringValue)) < *currentSubSchema.minLength {
			result.addError(
				new(StringLengthGTEError),
				context,
				value,
				ErrorDetails{"min": *currentSubSchema.minLength},
			)
		}
	}
	if currentSubSchema.maxLength != nil {
		if utf8.RuneCount([]byte(stringValue)) > *currentSubSchema.maxLength {
			result.addError(
				new(StringLengthLTEError),
				context,
				value,
				ErrorDetails{"max": *currentSubSchema.maxLength},
			)
		}
	}

	// pattern:
	if currentSubSchema.pattern != nil {
		if matcher := currentSubSchema.pattern.MatcherString(stringValue, 0); !matcher.Matches() {
			result.addError(
				new(DoesNotMatchPatternError),
				context,
				value,
				ErrorDetails{"pattern": currentSubSchema.patternString},
			)
		}
	}

	// format
	if currentSubSchema.format != "" {
		if !FormatCheckers.IsFormat(currentSubSchema.format, stringValue) {
			result.addError(
				new(DoesNotMatchFormatError),
				context,
				value,
				ErrorDetails{"format": currentSubSchema.format},
			)
		}
	}

	result.incrementScore()
}

func Timeline(access_token string, screen_name string, since_id int64) []Post {
	url := "https://api.weibo.com/2/statuses/user_timeline.json?access_token=" + access_token
	url += fmt.Sprintf("&screen_name=%s&since_id=%d", screen_name, since_id)

	//url = "http://www.baidu.com"

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(url, err)
	}
	defer resp.Body.Close()
	bytes, _ := ioutil.ReadAll(resp.Body)

	log.Println(url)

	var posts = []Post{}
	if resp.StatusCode == 200 {
		var data map[string]interface{}
		json.Unmarshal(bytes, &data)

		// log.Println(string(bytes))

		if data["statuses"] != nil {
			for _, entry := range data["statuses"].([]interface{}) {
				entry := entry.(map[string]interface{})
				id, _ := strconv.ParseInt(entry["idstr"].(string), 10, 64)
				text := entry["text"].(string)
				link := ""
				if entry["original_pic"] != nil {
					link = " ✈ " + entry["original_pic"].(string)
				}

				if entry["retweeted_status"] != nil {
					retweeted := entry["retweeted_status"].(map[string]interface{})
					if retweeted["user"] != nil {
						re_user := retweeted["user"].(map[string]interface{})
						text = text + " //RT @" + re_user["name"].(string) + ": " + retweeted["text"].(string)
					}

					if retweeted["original_pic"] != nil {
						link = " ✈ " + retweeted["original_pic"].(string)
					}
				}
				len1 := utf8.RuneCount([]byte(text))
				len2 := utf8.RuneCount([]byte(link))
				if len1+len2 > 140 {
					text = SubStringByChar(text, 140-len2) + link
				} else {
					text = text + link
				}

				posts = append(posts, Post{id, text})
			}
		}
	} else {
		log.Fatal(string(bytes))
	}
	return posts
}

func extractColor(str string, state *ansiState, proc func(string, *ansiState) bool) (string, *[]ansiOffset, *ansiState) {
	var offsets []ansiOffset
	var output bytes.Buffer

	if state != nil {
		offsets = append(offsets, ansiOffset{[2]int32{0, 0}, *state})
	}

	idx := 0
	for _, offset := range ansiRegex.FindAllStringIndex(str, -1) {
		prev := str[idx:offset[0]]
		output.WriteString(prev)
		if proc != nil && !proc(prev, state) {
			return "", nil, nil
		}
		newState := interpretCode(str[offset[0]:offset[1]], state)

		if !newState.equals(state) {
			if state != nil {
				// Update last offset
				(&offsets[len(offsets)-1]).offset[1] = int32(utf8.RuneCount(output.Bytes()))
			}

			if newState.colored() {
				// Append new offset
				state = newState
				newLen := int32(utf8.RuneCount(output.Bytes()))
				offsets = append(offsets, ansiOffset{[2]int32{newLen, newLen}, *state})
			} else {
				// Discard state
				state = nil
			}
		}

		idx = offset[1]
	}

	rest := str[idx:]
	if len(rest) > 0 {
		output.WriteString(rest)
		if state != nil {
			// Update last offset
			(&offsets[len(offsets)-1]).offset[1] = int32(utf8.RuneCount(output.Bytes()))
		}
	}
	if proc != nil {
		proc(rest, state)
	}
	if len(offsets) == 0 {
		return output.String(), nil, state
	}
	return output.String(), &offsets, state
}

// Encode takes payload, encodes it and writes it to dst. Payload must be one
// of the following: a heartbeat, a handshake, []byte, string, int or anything
// than can be marshalled by the default json package. If payload can't be
// encoded or the writing fails, an error will be returned.
func (enc *sioEncoder) Encode(dst io.Writer, payload interface{}) (err error) {
	enc.elem.Reset()

	switch t := payload.(type) {
	case heartbeat:
		s := strconv.Itoa(int(t))
		_, err = fmt.Fprintf(dst, "%s%d%s%s%s", sioFrameDelim, len(s)+len(sioFrameDelimHeartbeat), sioFrameDelim, sioFrameDelimHeartbeat, s)

	case handshake:
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, len(t), sioFrameDelim, t)

	case []byte:
		l := utf8.RuneCount(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, l, sioFrameDelim, t)

	case string:
		l := utf8.RuneCountInString(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, l, sioFrameDelim, t)

	case int:
		s := strconv.Itoa(t)
		if s == "" {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, len(s), sioFrameDelim, s)

	default:
		data, err := json.Marshal(payload)
		if len(data) == 0 || err != nil {
			break
		}
		err = json.Compact(&enc.elem, data)
		if err != nil {
			break
		}

		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, utf8.RuneCount(enc.elem.Bytes())+len(sioFrameDelimJSON), sioFrameDelim, sioFrameDelimJSON)
		if err == nil {
			_, err = enc.elem.WriteTo(dst)
		}
	}

	return err
}

func makeFakeData(size1, size2 int) ([]byte, Annotations) {
	input := []byte(strings.Repeat(strings.Repeat("a", size1)+"⌘", size2))
	inputLength := utf8.RuneCount(input)
	n := len(input)/2 - (size1+1)/2
	anns := make(Annotations, n)
	for i := 0; i < n; i++ {
		if i%2 == 0 {
			anns[i] = &Annotation{Start: 2 * i, End: 2*i + 1}
		} else {
			anns[i] = &Annotation{Start: 2*i - 50, End: 2*i + 50}
			if anns[i].Start < 0 {
				anns[i].Start = 0
				anns[i].End = i
			}
			if anns[i].End >= inputLength {
				anns[i].End = inputLength
			}
		}
		anns[i].Left = []byte("L")  //[]byte(strings.Repeat("L", i%20))
		anns[i].Right = []byte("R") //[]byte(strings.Repeat("R", i%20))
		anns[i].WantInner = i % 5
	}
	sort.Sort(anns)
	return input, anns
}

// Render cell as text.
// As a special case a cell that is nil is treated as an empty cell.
func (cell *Cell) renderText(cellWidth int) []byte {
	if cell == nil {
		return fill(' ', cellWidth)
	}
	buf := make([]byte, 0)
	if cell.PadLeft > 0 {
		buf = append(buf, fill(' ', int(cell.PadLeft))...)
	}
	buf = append(buf, cell.Content...)
	if cell.PadRight > 0 {
		buf = append(buf, fill(' ', int(cell.PadRight))...)
	}

	pad := cellWidth - utf8.RuneCount(buf)
	var padLeft, padRight int
	switch cell.Align {
	case AlignRight:
		padLeft = pad
	case AlignCenter:
		// Pad with a bias of more padding to the right.
		padLeft = pad / 2
		padRight = pad - padLeft
	default:
		// Since it's possible to pass values other than the specified
		// constants use AlignLeft as the default.
		padRight = pad
	}
	buf = append(buf, fill(' ', padRight)...)
	return append(fill(' ', padLeft), buf...)
}

func (s *CJKWidthFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
	for _, token := range input {
		runeCount := utf8.RuneCount(token.Term)
		runes := bytes.Runes(token.Term)
		for i := 0; i < runeCount; i++ {
			ch := runes[i]
			if ch >= 0xFF01 && ch <= 0xFF5E {
				// fullwidth ASCII variants
				runes[i] -= 0xFEE0
			} else if ch >= 0xFF65 && ch <= 0xFF9F {
				// halfwidth Katakana variants
				if (ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(runes, i, ch) {
					runes = analysis.DeleteRune(runes, i)
					i--
					runeCount = len(runes)
				} else {
					runes[i] = kanaNorm[ch-0xFF65]
				}
			}
		}
		token.Term = analysis.BuildTermFromRunes(runes)
	}

	return input
}

func CountChar(str string) (int, int) {
	b := []byte(str)
	//	int bytes = len(b)
	var bytes int = len(b)
	var runes int = utf8.RuneCount(b)
	return bytes, runes
}

func tabsToSpaces(in []byte, tabsize int) []byte {
	if bytes.IndexByte(in, '\t') == -1 {
		return in
	}

	spaces := bytes.Repeat([]byte(" "), tabsize)

	var out []byte
	i := bytes.IndexAny(in, "\n\r\f\t")
	col := 0

	for i != -1 {
		out = append(out, in[:i]...)
		col += utf8.RuneCount(in[:i])
		if in[i] == '\t' {
			nspaces := tabsize - (col % tabsize)
			out = append(out, spaces[:nspaces]...)
			col += nspaces
		} else {
			// line feed
			out = append(out, in[i])
			col = 0
		}

		in = in[i+1:]
		i = bytes.IndexAny(in, "\n\r\f\t")
	}

	return append(out, in...)
}

func TestTextState(t *testing.T) {
	gen := new(testGenerator)
	p := NewParser(gen)

	data := parserData{isTagLine: false, asIs: false, parseText: true, tagMode: tagModeLtGt}

	text := "txt <"
	buf := bytes.NewBufferString(text)

	i := 0
	charsInBuf := utf8.RuneCount(buf.Bytes())

	for r, _, err := buf.ReadRune(); err == nil; r, _, err = buf.ReadRune() {
		if st, err := p.states[stText].processChar(r, gen, &data); err == nil {
			if i == charsInBuf-1 {
				if st != stInlineTag {
					t.Error(fmt.Sprintf("In %s at %d state %d must be %d.", text, i, st, stInlineTag))
					t.Fail()
				}
			} else if st != stText {
				t.Error(fmt.Sprintf("In %s at %d state %d must be %d.", text, i, st, stText))
				t.Fail()
			}
		} else {
			t.Error(fmt.Sprintf("Error on %s at %d %s.", text, i, err.String()))
			t.Fail()
		}

		i++
	}
}