Golang utf8.RuneLen函数代码示例

func ExampleRuneLen() {
	fmt.Println(utf8.RuneLen('a'))
	fmt.Println(utf8.RuneLen('界'))
	// Output:
	// 1
	// 3
}

// RankMatch is similar to Match except it will measure the Levenshtein
// distance between the source and the target and return its result. If there
// was no match, it will return -1.
// Given the requirements of match, RankMatch only needs to perform a subset of
// the Levenshtein calculation, only deletions need be considered, required
// additions and substitutions would fail the match test.
func RankMatch(source, target string) int {
	lenDiff := len(target) - len(source)

	if lenDiff < 0 {
		return -1
	}

	if lenDiff == 0 && source == target {
		return 0
	}

	runeDiff := 0

Outer:
	for _, r1 := range source {
		for i, r2 := range target {
			if r1 == r2 {
				target = target[i+utf8.RuneLen(r2):]
				continue Outer
			} else {
				runeDiff++
			}
		}
		return -1
	}

	// Count up remaining char
	for len(target) > 0 {
		target = target[utf8.RuneLen(rune(target[0])):]
		runeDiff++
	}

	return runeDiff
}

// normalizeBidi attempts to prevent names from using bidi control codes to
// screw up our layout
func normalizeBidi(name string) string {
	bidiExplicitDepth := 0
	bidiIsolateDepth := 0

	for _, c := range name {
		switch c {
		case ltrEmbed, rtlEmbed, ltrOverride, rtlOverride:
			bidiExplicitDepth++
		case bidiExplicitPop:
			bidiExplicitDepth--
		case ltrIsolate, rtlIsolate, fsIsolate:
			bidiIsolateDepth++
		case bidiIsolatePop:
			bidiIsolateDepth--
		}
	}
	if bidiExplicitDepth+bidiIsolateDepth > 0 {
		pops := make([]byte,
			bidiExplicitDepth*utf8.RuneLen(bidiExplicitPop)+bidiIsolateDepth+utf8.RuneLen(bidiIsolatePop))
		i := 0
		for ; bidiExplicitDepth > 0; bidiExplicitDepth-- {
			i += utf8.EncodeRune(pops[i:], bidiExplicitPop)
		}
		for ; bidiIsolateDepth > 0; bidiIsolateDepth-- {
			i += utf8.EncodeRune(pops[i:], bidiIsolatePop)
		}
		return name + string(pops[:i])
	}
	return name
}

func reverse(s []byte) []byte {
	rest := subslice{s, 0, len(s)}
	result := subslice{s, 0, len(s)}
	// note: que は高々サイズ4の []rune
	que := queue{}
	for !rest.empty() {
		// 後ろから要素を取り出して...
		r := rest.popBack()
		// 先頭に十分な空きができるまで先頭の要素をキューに追加
		for !rest.empty() && frontInsertSpace(rest, result) < utf8.RuneLen(r) {
			que.push(rest.popFront())
		}
		// 後ろから取り出した要素を先頭に移動
		result.pushFront(r)
		// 先頭から取り出した要素を詰めれるだけ後ろに逆順で詰める
		for len(que) > 0 {
			if backInsertSpace(rest, result) < utf8.RuneLen(que.front()) {
				break
			}
			result.pushBack(que.front())
			que.pop()
		}
	}
	// 取り出せる要素が無くなったら，キューの要素を余った隙間に後ろに逆順で追加
	for len(que) > 0 {
		result.pushBack(que.front())
		que.pop()
	}
	return s
}

func TestCharcount(t *testing.T) {
	var tests = []struct {
		input   string
		counts  map[rune]int
		utflen  [utf8.UTFMax + 1]int
		invalid int
	}{
		{"Hello", map[rune]int{'H': 1, 'e': 1, 'l': 2, 'o': 1}, utflenFromMap(map[int]int{1: 5}), 0},
		{"あ", map[rune]int{'あ': 1}, utflenFromMap(map[int]int{utf8.RuneLen('あ'): 1}), 0},
		{
			"あiueお",
			map[rune]int{'あ': 1, 'i': 1, 'u': 1, 'e': 1, 'お': 1},
			utflenFromMap(map[int]int{utf8.RuneLen('あ'): 2, 1: 3}),
			0,
		},
	}

	for _, test := range tests {
		in := bufio.NewReader(strings.NewReader(test.input))
		counts, utflen, invalid, err := charcount(in)
		if !reflect.DeepEqual(counts, test.counts) || utflen != test.utflen || invalid != test.invalid || err != nil {
			t.Errorf("charcount(%q) = %v, %v, %v, %v ; want %v, %v, %v, nil", test.input,
				counts, utflen, invalid, err,
				test.counts, test.utflen, test.invalid)
		}
	}
}

// htmlReplacer returns s with runes replaced according to replacementTable
// and when badRunes is true, certain bad runes are allowed through unescaped.
func htmlReplacer(s string, replacementTable []string, badRunes bool) string {
	written, b := 0, new(bytes.Buffer)
	for i, r := range s {
		if int(r) < len(replacementTable) {
			if repl := replacementTable[r]; len(repl) != 0 {
				b.WriteString(s[written:i])
				b.WriteString(repl)
				// Valid as long as replacementTable doesn't
				// include anything above 0x7f.
				written = i + utf8.RuneLen(r)
			}
		} else if badRunes {
			// No-op.
			// IE does not allow these ranges in unquoted attrs.
		} else if 0xfdd0 <= r && r <= 0xfdef || 0xfff0 <= r && r <= 0xffff {
			fmt.Fprintf(b, "%s&#x%x;", s[written:i], r)
			written = i + utf8.RuneLen(r)
		}
	}
	if written == 0 {
		return s
	}
	b.WriteString(s[written:])
	return b.String()
}

// compileGlobPattern takes a given pattern string consisting of typical
// wildcard characters *, ?, or any literal string and returns a compiled slice
// of scanner functions.
//
// Any character in the pattern string can be escaped using a backslash to
// produce the literal character following it rather than a special character.
func compileGlobPattern(pattern string) ([]*globScanner, error) {
	// compile scanner function array
	wildcards := make([]*globScanner, 0, 4)
	for index, code := range pattern {
		var fn scanFunc = nil
		var start int = -1
		var kind globKind
		switch {
		case code == '\\':
			fn = consumeSubstring
			kind = globString
		case code == '*':
			fn = consumeAllPreceding
			kind = globMany
		case code == '?':
			fn = consumeOnePreceding
			kind = globOne
		case index == 0:
			fn = consumeSubstring
			start = index
			kind = globString
		default:
			continue
		}

		numWildcards := len(wildcards)
		if numWildcards > 0 {
			last := wildcards[numWildcards-1]
			if (kind == globOne || kind == globMany) && last.kind == globMany && last.start == index {
				return nil, ErrInvalidGlobSequence
			} else if code == '\\' && len(last.substr) == 0 {
				last.start += utf8.RuneLen(code)
				continue
			} else {
				last.substr = pattern[last.start:index]
			}
		}

		if start == -1 {
			start = index + utf8.RuneLen(code)
		}

		wildcards = append(wildcards, &globScanner{fn, kind, "", start})
	}

	numWildcards := len(wildcards)
	if numWildcards > 0 {
		last := wildcards[numWildcards-1]
		last.substr = pattern[last.start:]
	}

	wildcards = append(wildcards, &globScanner{consumeEnd, globEnd, "", len(pattern)})

	return wildcards, nil
}

// Write a rune to the underlying slice. If the rune is invalid, then the
// RuneError symbol is written. The rune is only written if there is available
// buffer space, otherwise ErrShortWrite is returned.
func (w *Writer) WriteRune(r rune) (cnt int, err error) {
	cnt = utf8.RuneLen(r)
	if cnt == -1 {
		r = utf8.RuneError
		cnt = utf8.RuneLen(r)
	}
	if availCnt := int64(len(w.buf)) - w.idx; availCnt < int64(cnt) {
		return 0, io.ErrShortWrite
	}
	cnt = utf8.EncodeRune(w.buf[w.idx:], r)
	w.idx += int64(cnt)
	return cnt, nil
}

/**
 * Lexer::BackupRunes
 */
func (l *lexer) BackupRunes(n int) {
	for ; n > 0; n-- {
		if l.pos > 0 {
			l.pos--
			i := l.runes.Peek(l.pos) // 0-based
			r := i.(rune)
			l.tokenLen -= utf8.RuneLen(r)
			l.column -= utf8.RuneLen(r)
		} else {
			panic("Underflow Exception")
		}
	}
}

func (p *parser) handlePreEscape(char rune) {
	switch char {
	case '[':
		p.instructionStartedAt = p.cursor + utf8.RuneLen('[')
		p.instructions = make([]string, 0, 1)
		p.mode = MODE_ESCAPE
	case ']':
		p.instructionStartedAt = p.cursor + utf8.RuneLen('[')
		p.mode = MODE_ITERM_ESCAPE
	default:
		// Not an escape code, false alarm
		p.cursor = p.escapeStartedAt
		p.mode = MODE_NORMAL
	}
}

func (self *_lexer) skip(count int) {
	read := self.readIn[self.tail : self.tail+count]
	for _, chr := range read {
		self.tail += 1
		self.tailOffset += utf8.RuneLen(chr)
	}
}

func (p *Parser) parseShort(s *parseState, optname string, argument *string) (option *Option, err error) {
	if argument == nil {
		optname, argument = p.splitShortConcatArg(s, optname)
	}

	for i, c := range optname {
		shortname := string(c)

		if option = s.lookup.shortNames[shortname]; option != nil {
			// Only the last short argument can consume an argument from
			// the arguments list, and only if it's non optional
			canarg := (i+utf8.RuneLen(c) == len(optname)) && !option.OptionalArgument

			if _, err := p.parseOption(s, shortname, option, canarg, argument); err != nil {
				return option, err
			}
		} else {
			return nil, newError(ErrUnknownFlag, fmt.Sprintf("unknown flag `%s'", shortname))
		}

		// Only the first option can have a concatted argument, so just
		// clear argument here
		argument = nil
	}

	return option, nil
}

func main() {
	s := "¶ Greetings!"
	r, l := utf8.DecodeRuneInString(s)
	l2 := utf8.RuneLen(r)
	ok := utf8.ValidString(s)
	fmt.Printf("rune %c length %d = %d ok %t\n", r, l, l2, ok)
}

func splitPathOnSeparator(path string, separator rune) []string {
	// if the separator is '\\', then we can just split...
	if separator == '\\' {
		return strings.Split(path, string(separator))
	}

	// otherwise, we need to be careful of situations where the separator was escaped
	cnt := strings.Count(path, string(separator))
	if cnt == 0 {
		return []string{path}
	}
	ret := make([]string, cnt+1)
	pathlen := len(path)
	separatorLen := utf8.RuneLen(separator)
	idx := 0
	for start := 0; start < pathlen; {
		end := indexRuneWithEscaping(path[start:], separator)
		if end == -1 {
			end = pathlen
		} else {
			end += start
		}
		ret[idx] = path[start:end]
		start = end + separatorLen
		idx++
	}
	return ret[:idx]
}

// processEscape processes a single escape sequence and returns number of bytes processed.
func (r *Lexer) processEscape(data []byte) (int, error) {
	if len(data) < 2 {
		return 0, fmt.Errorf("syntax error at %v", string(data))
	}

	c := data[1]
	switch c {
	case '"', '/', '\\':
		r.token.byteValue = append(r.token.byteValue, c)
		return 2, nil
	case 'b':
		r.token.byteValue = append(r.token.byteValue, '\b')
		return 2, nil
	case 'f':
		r.token.byteValue = append(r.token.byteValue, '\f')
		return 2, nil
	case 'n':
		r.token.byteValue = append(r.token.byteValue, '\n')
		return 2, nil
	case 'r':
		r.token.byteValue = append(r.token.byteValue, '\r')
		return 2, nil
	case 't':
		r.token.byteValue = append(r.token.byteValue, '\t')
		return 2, nil
	case 'u':
	default:
		return 0, fmt.Errorf("syntax error")
	}

	var val rune

	for i := 2; i < len(data) && i < 6; i++ {
		var v byte
		c = data[i]
		switch c {
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			v = c - '0'
		case 'a', 'b', 'c', 'd', 'e', 'f':
			v = c - 'a' + 10
		case 'A', 'B', 'C', 'D', 'E', 'F':
			v = c - 'A' + 10
		default:
			return 0, fmt.Errorf("syntax error")
		}

		val <<= 4
		val |= rune(v)
	}

	l := utf8.RuneLen(val)
	if l == -1 {
		return 0, fmt.Errorf("invalid unicode escape")
	}

	var d [4]byte
	utf8.EncodeRune(d[:], val)
	r.token.byteValue = append(r.token.byteValue, d[:l]...)
	return 6, nil
}