Golang utf8.EncodeRune函数代码示例

func TestReadWriteRune(t *testing.T) {
	const NRune = 1000
	byteBuf := new(bytes.Buffer)
	w := NewWriter(byteBuf)
	// Write the runes out using WriteRune
	buf := make([]byte, utf8.UTFMax)
	for rune := 0; rune < NRune; rune++ {
		size := utf8.EncodeRune(rune, buf)
		nbytes, err := w.WriteRune(rune)
		if err != nil {
			t.Fatalf("WriteRune(0x%x) error: %s", rune, err)
		}
		if nbytes != size {
			t.Fatalf("WriteRune(0x%x) expected %d, got %d", rune, size, nbytes)
		}
	}
	w.Flush()

	r := NewReader(byteBuf)
	// Read them back with ReadRune
	for rune := 0; rune < NRune; rune++ {
		size := utf8.EncodeRune(rune, buf)
		nr, nbytes, err := r.ReadRune()
		if nr != rune || nbytes != size || err != nil {
			t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err)
		}
	}
}

func TestRuneIO(t *testing.T) {
	const NRune = 1000
	// Built a test array while we write the data
	b := make([]byte, utf8.UTFMax*NRune)
	var buf Buffer
	n := 0
	for r := 0; r < NRune; r++ {
		size := utf8.EncodeRune(r, b[n:])
		nbytes, err := buf.WriteRune(r)
		if err != nil {
			t.Fatalf("WriteRune(0x%x) error: %s", r, err)
		}
		if nbytes != size {
			t.Fatalf("WriteRune(0x%x) expected %d, got %d", r, size, nbytes)
		}
		n += size
	}
	b = b[0:n]

	// Check the resulting bytes
	if !Equal(buf.Bytes(), b) {
		t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b)
	}

	// Read it back with ReadRune
	for r := 0; r < NRune; r++ {
		size := utf8.EncodeRune(r, b)
		nr, nbytes, err := buf.ReadRune()
		if nr != r || nbytes != size || err != nil {
			t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err)
		}
	}
}

func (p *Trie) outputDot(vec *vector.StringVector, rune int, serial int64, rgen *rand.Rand) {
	this := make([]byte, 10)
	child := make([]byte, 10)

	utf8.EncodeRune(this, rune)

	thisChar := string(this[0])

	if serial == -1 {
		thisChar = "root"
	}

	for childRune, childNode := range p.children {
		utf8.EncodeRune(child, childRune)
		childSerial := rgen.Int63()
		childNodeStr := fmt.Sprintf("\"%s(%d)\"", string(child[0]), childSerial)
		var notation string

		if string(child[0]) == "/" {
			notation = fmt.Sprintf("[label=\"%s\" shape=box color=red]", string(child[0]))
		} else {
			notation = fmt.Sprintf("[label=\"%s\"]", string(child[0]))
		}
		vec.Push(fmt.Sprintf("\t%s %s\n\t\"%s(%d)\" -> \"%s(%d)\"", childNodeStr, notation, thisChar, serial, string(child[0]), childSerial))
		childNode.outputDot(vec, childRune, childSerial, rgen)
	}
}

func TestRuneIO(t *testing.T) {
	const NRune = 1000
	// Built a test array while we write the data
	b := make([]byte, utf8.UTFMax*NRune)
	var buf Buffer
	n := 0
	for r := rune(0); r < NRune; r++ {
		size := utf8.EncodeRune(b[n:], r)
		nbytes, err := buf.WriteRune(r)
		if err != nil {
			t.Fatalf("WriteRune(%U) error: %s", r, err)
		}
		if nbytes != size {
			t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes)
		}
		n += size
	}
	b = b[0:n]

	// Check the resulting bytes
	if !Equal(buf.Bytes(), b) {
		t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b)
	}

	p := make([]byte, utf8.UTFMax)
	// Read it back with ReadRune
	for r := rune(0); r < NRune; r++ {
		size := utf8.EncodeRune(p, r)
		nr, nbytes, err := buf.ReadRune()
		if nr != r || nbytes != size || err != nil {
			t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err)
		}
	}

	// Check that UnreadRune works
	buf.Reset()
	buf.Write(b)
	for r := rune(0); r < NRune; r++ {
		r1, size, _ := buf.ReadRune()
		if err := buf.UnreadRune(); err != nil {
			t.Fatalf("UnreadRune(%U) got error %q", r, err)
		}
		r2, nbytes, err := buf.ReadRune()
		if r1 != r2 || r1 != r || nbytes != size || err != nil {
			t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err)
		}
	}
}

// WriteRune writes a single Unicode code point, returning
// the number of bytes written and any error.
func (b *Writer) WriteRune(r rune) (size int, err error) {
	if r < utf8.RuneSelf {
		err = b.WriteByte(byte(r))
		if err != nil {
			return 0, err
		}
		return 1, nil
	}
	if b.err != nil {
		return 0, b.err
	}
	n := b.Available()
	if n < utf8.UTFMax {
		if b.Flush(); b.err != nil {
			return 0, b.err
		}
		n = b.Available()
		if n < utf8.UTFMax {
			// Can only happen if buffer is silly small.
			return b.WriteString(string(r))
		}
	}
	size = utf8.EncodeRune(b.buf[b.n:], r)
	b.n += size
	return size, nil
}

// Converts a single numerical html entity to a regular Go utf8-token.
func EntityToUtf8(entity string) string {
	var ok bool
	if ok = reg_entnamed.MatchString(entity); ok {
		return namedEntityToUtf8(entity[1 : len(entity)-1])
	}

	if ok = reg_entnumeric.MatchString(entity); !ok {
		return "&" + entity[2:len(entity)-1] + ";"
	}

	var err os.Error
	var num int

	entity = entity[2 : len(entity)-1]
	if num, err = strconv.Atoi(entity); err != nil {
		return "&#" + entity + ";"
	}

	var arr [4]byte
	if size := utf8.EncodeRune(arr[:], num); size == 0 {
		return "&#" + entity + ";"
	}

	return string(arr[:])
}

// unescapeEntity reads an entity like "<" from b[src:] and writes the
// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
// Precondition: src[0] == '&' && dst <= src.
func unescapeEntity(b []byte, dst, src int) (dst1, src1 int) {
	// TODO(nigeltao): Check that this entity substitution algorithm matches the spec:
	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
	// TODO(nigeltao): Handle things like "&#20013;" or "&#x4e2d;".

	// i starts at 1 because we already know that s[0] == '&'.
	i, s := 1, b[src:]
	for i < len(s) {
		c := s[i]
		i++
		// Lower-cased characters are more common in entities, so we check for them first.
		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
			continue
		}
		if c != ';' {
			i--
		}
		x := entity[string(s[1:i])]
		if x != 0 {
			return dst + utf8.EncodeRune(x, b[dst:]), src + i
		}
		break
	}
	dst1, src1 = dst+i, src+i
	copy(b[dst:dst1], b[src:src1])
	return dst1, src1
}

// Extract regular text from the beginning of the pattern,
// possibly after a leading iBOT.
// That text can be used by doExecute to speed up matching.
func (re *Regexp) setPrefix() {
	var b []byte
	var utf = make([]byte, utf8.UTFMax)
	var inst *instr
	// First instruction is start; skip that.  Also skip any initial iBOT.
	inst = re.inst[0].next
	for inst.kind == iBOT {
		inst = inst.next
	}
Loop:
	for ; inst.kind != iEnd; inst = inst.next {
		// stop if this is not a char
		if inst.kind != iChar {
			break
		}
		// stop if this char can be followed by a match for an empty string,
		// which includes closures, ^, and $.
		switch inst.next.kind {
		case iBOT, iEOT, iAlt:
			break Loop
		}
		n := utf8.EncodeRune(utf, inst.char)
		b = append(b, utf[0:n]...)
	}
	// point prefixStart instruction to first non-CHAR after prefix
	re.prefixStart = inst
	re.prefixBytes = b
	re.prefix = string(b)
}

// Map returns a copy of the byte array s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement.  The characters in s and the
// output are interpreted as UTF-8-encoded Unicode code points.
func Map(mapping func(rune int) int, s []byte) []byte {
	// In the worst case, the array can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for i := 0; i < len(s); {
		wid := 1
		rune := int(s[i])
		if rune >= utf8.RuneSelf {
			rune, wid = utf8.DecodeRune(s[i:])
		}
		rune = mapping(rune)
		if rune >= 0 {
			if nbytes+utf8.RuneLen(rune) > maxbytes {
				// Grow the buffer.
				maxbytes = maxbytes*2 + utf8.UTFMax
				nb := make([]byte, maxbytes)
				copy(nb, b[0:nbytes])
				b = nb
			}
			nbytes += utf8.EncodeRune(b[nbytes:maxbytes], rune)
		}
		i += wid
	}
	return b[0:nbytes]
}

// Map returns a copy of the string s with all its characters modified
// according to the mapping function.
func Map(mapping func(rune int) int, s string) string {
	// In the worst case, the string can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for _, c := range s {
		rune := mapping(c)
		wid := 1
		if rune >= utf8.RuneSelf {
			wid = utf8.RuneLen(rune)
		}
		if nbytes+wid > maxbytes {
			// Grow the buffer.
			maxbytes = maxbytes*2 + utf8.UTFMax
			nb := make([]byte, maxbytes)
			for i, c := range b[0:nbytes] {
				nb[i] = c
			}
			b = nb
		}
		nbytes += utf8.EncodeRune(rune, b[nbytes:maxbytes])
	}
	return string(b[0:nbytes])
}

// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
func (rb *reorderBuffer) appendRune(rune uint32) {
	bn := rb.nbyte
	sz := utf8.EncodeRune(rb.byte[bn:], int(rune))
	rb.nbyte += uint8(sz)
	rb.rune[rb.nrune] = runeInfo{bn, uint8(sz), 0, 0}
	rb.nrune++
}

// Inserts a character in the cursor position.
func (b *buffer) insertRune(rune int) os.Error {
	var useRefresh bool

	b.grow(b.size + 1) // Check if there is free space for one more character

	// Avoid a full update of the line.
	if b.pos == b.size {
		char := make([]byte, utf8.UTFMax)
		utf8.EncodeRune(char, rune)

		if _, err := output.Write(char); err != nil {
			return outputError(err.String())
		}
	} else {
		useRefresh = true
		copy(b.data[b.pos+1:b.size+1], b.data[b.pos:b.size])
	}

	b.data[b.pos] = rune
	b.pos++
	b.size++

	if useRefresh {
		return b.refresh()
	}
	return nil
}

// Returns a slice of the contents of the buffer.
func (b *buffer) toBytes() []byte {
	chars := make([]byte, b.size*utf8.UTFMax)
	var end, runeLen int

	// === Each character (as integer) is encoded to []byte
	for i := 0; i < b.size; i++ {
		if i != 0 {
			runeLen = utf8.EncodeRune(chars[end:], b.data[i])
			end += runeLen
		} else {
			runeLen = utf8.EncodeRune(chars, b.data[i])
			end = runeLen
		}
	}
	return chars[:end]
}

// Extract regular text from the beginning of the pattern.
// That text can be used by doExecute to speed up matching.
func (re *Regexp) setPrefix() {
	var b []byte
	var utf = make([]byte, utf8.UTFMax)
	// First instruction is start; skip that.
	i := re.inst.At(0).(instr).next().index()
Loop:
	for i < re.inst.Len() {
		inst := re.inst.At(i).(instr)
		// stop if this is not a char
		if inst.kind() != _CHAR {
			break
		}
		// stop if this char can be followed by a match for an empty string,
		// which includes closures, ^, and $.
		switch re.inst.At(inst.next().index()).(instr).kind() {
		case _BOT, _EOT, _ALT:
			break Loop
		}
		n := utf8.EncodeRune(inst.(*_Char).char, utf)
		b = bytes.Add(b, utf[0:n])
		i = inst.next().index()
	}
	// point prefixStart instruction to first non-CHAR after prefix
	re.prefixStart = re.inst.At(i).(instr)
	re.prefixBytes = b
	re.prefix = string(b)
}

// fmtC formats a rune for the 'c' format.
func (p *pp) fmtC(c int64) {
	rune := int(c) // Check for overflow.
	if int64(rune) != c {
		rune = utf8.RuneError
	}
	w := utf8.EncodeRune(p.runeBuf[0:utf8.UTFMax], rune)
	p.fmt.pad(p.runeBuf[0:w])
}