Golang utf8.DecodeRune函数代码示例

func (t *Trie) Add(name string, data []int) {
	nm := []byte(name)
	if t.head == nil {
		t.init()
	}
	cur := t.head

	i := 0
	for i < len(nm) {
		r, size := utf8.DecodeRune(nm[i:])
		if _, ok := cur.children[r]; ok {
			cur = cur.children[r]
			i += size
		} else {
			break
		}
	}

	for i < len(nm) {
		r, size := utf8.DecodeRune(nm[i:])
		if i+size <= len(nm) {
			if _, ok := cur.children[r]; !ok && cur.name != name {
				cur.children[r] = &node{name[:i+size], nil, make(map[rune]*node)}
				cur = cur.children[r]
				i += size
			}
		}
	}

	cur.val = data
}

func scanStmts(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	end := start
	// Scan until semicolon, marking end of statement.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if r == ';' {
			return i + width, data[start:i], nil
		} else if !unicode.IsSpace(r) {
			end = i + 1
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated statement. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:end], nil
	}
	// Request more data.
	return 0, nil, nil
}

// bufio.Scanner function to split data by words and quoted strings
func scanStrings(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}

	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Scan until space, marking end of word.
	inquote := false
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if r == '"' {
			inquote = !inquote
			continue
		}
		if unicode.IsSpace(r) && !inquote {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return 0, nil, nil
}

// scanWordsKeepPrefix is a split function for a Scanner that returns each
// space-separated word of text, with prefixing spaces included. It will never
// return an empty string. The definition of space is set by unicode.IsSpace.
//
// Adapted from bufio.ScanWords().
func scanTokensKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}
	if atEOF && len(data) == 0 || start == len(data) {
		return len(data), data, nil
	}
	if len(data) > start && data[start] == '#' {
		return scanLinesKeepPrefix(data, atEOF)
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if unicode.IsSpace(r) {
			return i, data[:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data, nil
	}
	// Request more data.
	return 0, nil, nil
}

// scanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func scanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}
	quote := false
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		switch {
		case i == 0 && r == '"':
			quote = true
		case !quote && unicode.IsSpace(r):
			return i + width, data[start:i], nil
		case quote && r == '"':
			return i + width, data[start+width : i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return start, nil, nil
}

func iter_words(data []byte, cb func(word []byte)) {
	for {
		if len(data) == 0 {
			return
		}

		r, rlen := utf8.DecodeRune(data)
		// skip non-word runes
		for !is_word(r) {
			data = data[rlen:]
			if len(data) == 0 {
				return
			}
			r, rlen = utf8.DecodeRune(data)
		}

		// must be on a word rune
		i := 0
		for is_word(r) && i < len(data) {
			i += rlen
			r, rlen = utf8.DecodeRune(data[i:])
		}
		cb(data[:i])
		data = data[i:]
	}
}

// pat = EscapeNormalString(pat)
func EscapeNormalString(in string) (rv string) {
	rv = ""
	var c rune
	var sz int

	for i := 0; i < len(in); i += sz {
		c, sz = utf8.DecodeRune([]byte(in[i:]))
		if c == '\\' {
			i += sz
			c, sz = utf8.DecodeRune([]byte(in[i:]))
			switch c {
			case 'n':
				rv += "\n"
			case 't':
				rv += "\t"
			case 'f':
				rv += "\f"
			case 'r':
				rv += "\r"
			case 'v':
				rv += "\v"
			default:
				rv += string(c)
			}
		} else {
			rv += string(c)
		}
	}
	return
}

// ScanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSpace(r) {
			break
		}
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSpace(r) {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return start, nil, nil
}

func (self *WordDict) AddReplaceRule(rule []byte) {
	if utf8.RuneCount(rule) != 2 {
		self.Panic("rule format differs from '=xX'")
	}
	from, fromSize := utf8.DecodeRune(rule)
	to, _ := utf8.DecodeRune(rule[fromSize:])
	self.runeMapping[from] = to
}

func init() {
	// setup the required runes
	colon, _ = utf8.DecodeRune([]byte(":"))
	dash, _ = utf8.DecodeRune([]byte("-"))
	period, _ = utf8.DecodeRune([]byte("."))
	slash, _ = utf8.DecodeRune([]byte("/"))
	underscore, _ = utf8.DecodeRune([]byte("_"))
}

func (lr *lexlReader) ScanRune(read bool) (rune, error) {
	fmt.Println("SCAN RUNE")
	if lr.size < 4 {
		fmt.Println(" < 4 fill")
		err := lr.attemptFill()
		if err != nil {
			fmt.Println("SCAN ERR")
			return 0, err
		}
		fmt.Println(" < 4 fill done")
	}
	fmt.Printf("lr.size: %d\n", lr.size)
	if lr.size == 0 {
		return 0, io.EOF
	}
	if len(lr.buf)-lr.pos < 4 {
		fmt.Println("END BUFFERING")
		nbuf := make([]byte, 4)
		nlen := 4
		if nlen > lr.size {
			nlen = lr.size
		}
		npos := lr.pos
		for i := 0; i < nlen; i++ {
			nbuf[i] = lr.buf[npos]
			npos++
			if npos >= len(lr.buf) {
				npos -= len(lr.buf)
			}
		}
		r, ns := utf8.DecodeRune(nbuf)
		if r == utf8.RuneError {
			return 0, errors.New("stream does not decode a utf-8 character")
		}
		if read {
			lr.pos += ns
			lr.size -= ns
			if lr.pos >= len(lr.buf) {
				lr.pos -= len(lr.buf)
			}
		}
		return r, nil
	}
	fmt.Println("DECODING FROM BUFFER")
	r, ns := utf8.DecodeRune(lr.buf[lr.pos:])
	if r == utf8.RuneError {
		return 0, errors.New("stream does not decode a utf-8 character")
	}
	if read {
		lr.pos += ns
		lr.size -= ns
		if lr.pos >= len(lr.buf) {
			lr.pos -= len(lr.buf)
		}
	}
	return r, nil
}

func CompareChars(word string) {
	s := []byte(word)
	for utf8.RuneCount(s) > 1 {
		r, size := utf8.DecodeRune(s)
		s = s[size:]
		nextR, size := utf8.DecodeRune(s)
		fmt.Print(r == nextR, ",")
	}
	fmt.Println()
}

func NewInput(in io.Reader) *Input {
	input := &Input{
		"",
		bufio.NewScanner(in),
	}

	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		//fmt.Println("Input:", string(data))
		// Skip leading spaces.
		start := 0
		for width := 0; start < len(data); start += width {
			var r rune
			r, width = utf8.DecodeRune(data[start:])
			if !IsWhitespace(r) {
				break
			}
		}
		if atEOF && len(data[start:]) == 0 {
			//fmt.Println("need more data 1")
			return 0, nil, nil
		}

		//fmt.Println("After WS Skip:", string(data[start:]))

		var r rune
		var width int
		r, width = utf8.DecodeRune(data[start:])
		if r == '(' || r == ')' {
			//fmt.Println("returning token:", string(data[start:start+width]))
			return start + width, data[start : start+width], nil
		}
		//fmt.Println("After paren check:", string(data[start:]))

		// Scan until space, marking end of word.
		for width, i := 0, start; i < len(data); i += width {
			r, width = utf8.DecodeRune(data[i:])
			//fmt.Printf("rune %d: %s\n", i, string(r))
			if IsWhitespace(r) || r == '(' || r == ')' {
				//fmt.Println("returning token:", string(data[start:i]))
				return i, data[start:i], nil
			}
		}
		// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
		if atEOF && len(data) > start {
			//fmt.Println("returning token:", string(data[start:]))
			return len(data) - start, data[start:], nil
		}
		// Request more data.
		//fmt.Println("need more data 2")
		return 0, nil, nil
	}
	input.Split(split)

	return input
}

func (src *Src) Consume(match ConsumeFunc) string {
	buf := src.Bytes()
	var m int
	for r, n := utf8.DecodeRune(buf); r != utf8.RuneError; r, n = utf8.DecodeRune(buf) {
		if !match(r) {
			break
		}
		buf = buf[n:]
		m += n
	}
	return src.SkipString(m)
}

// Creates a scanner that splits on words or quoted strings
func NewQuotedScanner(r io.Reader) *bufio.Scanner {
	scanner := bufio.NewScanner(r)
	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		// Skip leading spaces.
		start := 0
		for width := 0; start < len(data); start += width {
			var r rune
			r, width = utf8.DecodeRune(data[start:])
			if !unicode.IsSpace(r) {
				break
			}
		}

		// Does word start with a quote?
		quote, width := utf8.DecodeRune(data[start:])
		i := start
		if IsQuote(quote) {
			log.Debugf("Quote detected '%c'", quote)
			i = i + width
		} else {
			quote = 0

		}

		// Scan until space, marking end of word.
		for width := 0; i < len(data); i += width {
			var r rune
			r, width = utf8.DecodeRune(data[i:])
			if quote == 0 {
				if unicode.IsSpace(r) {
					return i + width, data[start:i], nil
				}
			} else {
				// Look for ending quote
				// BUG: need to implement escape handling
				if r == quote {
					log.Debugf("Found end quote %d chars after start", i)
					quote = 0
				}
			}
		}
		// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
		if atEOF && len(data) > start {
			return len(data), data[start:], nil
		}
		// Request more data.
		return start, nil, nil
	}
	scanner.Split(split)
	return scanner
}