Golang utf8.EncodeRune函数代码示例

func toCamelCase(x string) string {
	if len(x) == 0 {
		return ""
	}

	output := make([]byte, 0)
	uppercase := true

	for len(x) > 0 {
		v, size := utf8.DecodeRuneInString(x)

		// If underscore, append and keep going.
		if v == '_' {
			uppercase = true
		} else if unicode.IsLetter(v) {
			if uppercase {
				uppercase = false
				buf := make([]byte, size)
				utf8.EncodeRune(buf, unicode.ToUpper(v))
				output = bytes.Join([][]byte{output, buf}, nil)
			} else if unicode.IsUpper(v) {
				buf := make([]byte, size)
				utf8.EncodeRune(buf, v)
				output = bytes.Join([][]byte{output, buf}, []byte("_"))
			}
		}

		x = x[size:]
	}

	return string(output)
}

// normalizeBidi attempts to prevent names from using bidi control codes to
// screw up our layout
func normalizeBidi(name string) string {
	bidiExplicitDepth := 0
	bidiIsolateDepth := 0

	for _, c := range name {
		switch c {
		case ltrEmbed, rtlEmbed, ltrOverride, rtlOverride:
			bidiExplicitDepth++
		case bidiExplicitPop:
			bidiExplicitDepth--
		case ltrIsolate, rtlIsolate, fsIsolate:
			bidiIsolateDepth++
		case bidiIsolatePop:
			bidiIsolateDepth--
		}
	}
	if bidiExplicitDepth+bidiIsolateDepth > 0 {
		pops := make([]byte,
			bidiExplicitDepth*utf8.RuneLen(bidiExplicitPop)+bidiIsolateDepth+utf8.RuneLen(bidiIsolatePop))
		i := 0
		for ; bidiExplicitDepth > 0; bidiExplicitDepth-- {
			i += utf8.EncodeRune(pops[i:], bidiExplicitPop)
		}
		for ; bidiIsolateDepth > 0; bidiIsolateDepth-- {
			i += utf8.EncodeRune(pops[i:], bidiIsolatePop)
		}
		return name + string(pops[:i])
	}
	return name
}

func TestReadWriteRune(t *testing.T) {
	const NRune = 1000
	byteBuf := new(bytes.Buffer)
	w := NewAsyncWriter(byteBuf)
	// Write the runes out using WriteRune
	buf := make([]byte, utf8.UTFMax)
	for r := rune(0); r < NRune; r++ {
		size := utf8.EncodeRune(buf, r)
		nbytes, err := w.WriteRune(r)
		if err != nil {
			t.Fatalf("WriteRune(0x%x) error: %s", r, err)
		}
		if nbytes != size {
			t.Fatalf("WriteRune(0x%x) expected %d, got %d", r, size, nbytes)
		}
	}
	w.Flush()

	r := bufio.NewReader(byteBuf)
	// Read them back with ReadRune
	for r1 := rune(0); r1 < NRune; r1++ {
		size := utf8.EncodeRune(buf, r1)
		nr, nbytes, err := r.ReadRune()
		if nr != r1 || nbytes != size || err != nil {
			t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r1, nr, nbytes, r1, size, err)
		}
	}
}

// readConsole reads utf16 characters from console File,
// encodes them into utf8 and stores them in buffer buf.
// It returns the number of utf8 bytes read and an error, if any.
func (f *File) readConsole(buf []byte) (n int, err error) {
	if len(buf) == 0 {
		return 0, nil
	}
	if len(f.readbuf) > 0 {
		return f.copyReadConsoleBuffer(buf)
	}
	wchar, err := f.readOneUTF16FromConsole()
	if err != nil {
		return 0, err
	}
	r := rune(wchar)
	if utf16.IsSurrogate(r) {
		wchar, err := f.readOneUTF16FromConsole()
		if err != nil {
			return 0, err
		}
		r = utf16.DecodeRune(r, rune(wchar))
	}
	if nr := utf8.RuneLen(r); nr > len(buf) {
		start := len(f.readbuf)
		for ; nr > 0; nr-- {
			f.readbuf = append(f.readbuf, 0)
		}
		utf8.EncodeRune(f.readbuf[start:cap(f.readbuf)], r)
	} else {
		utf8.EncodeRune(buf, r)
		buf = buf[nr:]
		n += nr
	}
	if n > 0 {
		return n, nil
	}
	return f.copyReadConsoleBuffer(buf)
}

// ToUpperCamelCase returns a copy of the string s with all Unicode letters mapped to their camel case.
// It will convert to upper case previous letter of '_' and first letter, and remove letter of '_'.
func ToUpperCamelCase(s string) string {
	if s == "" {
		return ""
	}
	upper := true
	start := 0
	result := make([]byte, 0, len(s))
	var runeBuf [utf8.UTFMax]byte
	var initialism []byte
	for _, c := range s {
		if c == '_' {
			upper = true
			candidate := string(result[start:])
			initialism = initialism[:0]
			for _, r := range candidate {
				if r < utf8.RuneSelf {
					initialism = append(initialism, toUpperASCII(byte(r)))
				} else {
					n := utf8.EncodeRune(runeBuf[:], unicode.ToUpper(r))
					initialism = append(initialism, runeBuf[:n]...)
				}
			}
			if length := commonInitialism.LookupByBytes(initialism); length > 0 {
				result = append(result[:start], initialism...)
			}
			start = len(result)
			continue
		}
		if upper {
			if c < utf8.RuneSelf {
				result = append(result, toUpperASCII(byte(c)))
			} else {
				n := utf8.EncodeRune(runeBuf[:], unicode.ToUpper(c))
				result = append(result, runeBuf[:n]...)
			}
			upper = false
			continue
		}
		if c < utf8.RuneSelf {
			result = append(result, byte(c))
		} else {
			n := utf8.EncodeRune(runeBuf[:], c)
			result = append(result, runeBuf[:n]...)
		}
	}
	candidate := string(result[start:])
	initialism = initialism[:0]
	for _, r := range candidate {
		if r < utf8.RuneSelf {
			initialism = append(initialism, toUpperASCII(byte(r)))
		} else {
			n := utf8.EncodeRune(runeBuf[:], unicode.ToUpper(r))
			initialism = append(initialism, runeBuf[:n]...)
		}
	}
	if length := commonInitialism.LookupByBytes(initialism); length > 0 {
		result = append(result[:start], initialism...)
	}
	return string(result)
}

func UTF16ToUTF8Bytes(in []byte, out io.Writer) error {
	if len(in)%2 != 0 {
		return fmt.Errorf("input buffer must have an even length (length=%d)", len(in))
	}

	var runeBuf [4]byte
	var v1, v2 uint16
	for i := 0; i < len(in); i += 2 {
		v1 = uint16(in[i]) | uint16(in[i+1])<<8

		switch {
		case v1 < surr1, surr3 <= v1:
			n := utf8.EncodeRune(runeBuf[:], rune(v1))
			out.Write(runeBuf[:n])
		case surr1 <= v1 && v1 < surr2 && len(in) > i+2:
			v2 = uint16(in[i+2]) | uint16(in[i+3])<<8
			if surr2 <= v2 && v2 < surr3 {
				// valid surrogate sequence
				r := utf16.DecodeRune(rune(v1), rune(v2))
				n := utf8.EncodeRune(runeBuf[:], r)
				out.Write(runeBuf[:n])
			}
			i += 2
		default:
			// invalid surrogate sequence
			n := utf8.EncodeRune(runeBuf[:], replacementChar)
			out.Write(runeBuf[:n])
		}
	}

	return nil
}

func (t *Textbox) Tl(u rune, v rune) error {
	//Check if rune is valid
	if !utf8.ValidRune(v) {
		return errors.New("invalid rune.")
	}
	for i, val := range t.canvas {
		for j, w := 0, 0; j < len(val); j += w {
			runeValue, width := utf8.DecodeRune(val[j:])
			if runeValue == u {
				tmp := t.canvas[i][j : j+width]
				replaceWidth := utf8.RuneLen(v)
				if width == replaceWidth {
					// replace rune
					utf8.EncodeRune(tmp, v)
				} else if width > replaceWidth {
					// replace and pad
					utf8.EncodeRune(tmp, v)
					tmp = tmp[replaceWidth:]
					for len(tmp) > 0 {
						replaceWidth = utf8.EncodeRune(tmp, 0x0)
						tmp = tmp[replaceWidth:]
					}
				} else {
					// rune wont fit.
					return errors.New("rune too large.")
				}
			}
			w = width
		}
	}
	return nil
}

func TestFoldAgainstUnicode(t *testing.T) {
	const bufSize = 5
	buf1 := make([]byte, 0, bufSize)
	buf2 := make([]byte, 0, bufSize)
	var runes []rune
	for i := 0x20; i <= 0x7f; i++ {
		runes = append(runes, rune(i))
	}
	runes = append(runes, kelvin, smallLongEss)

	funcs := []struct {
		name   string
		fold   func(s, t []byte) bool
		letter bool // must be ASCII letter
		simple bool // must be simple ASCII letter (not 'S' or 'K')
	}{
		{
			name: "equalFoldRight",
			fold: equalFoldRight,
		},
		{
			name:   "asciiEqualFold",
			fold:   asciiEqualFold,
			simple: true,
		},
		{
			name:   "simpleLetterEqualFold",
			fold:   simpleLetterEqualFold,
			simple: true,
			letter: true,
		},
	}

	for _, ff := range funcs {
		for _, r := range runes {
			if r >= utf8.RuneSelf {
				continue
			}
			if ff.letter && !isASCIILetter(byte(r)) {
				continue
			}
			if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') {
				continue
			}
			for _, r2 := range runes {
				buf1 := append(buf1[:0], 'x')
				buf2 := append(buf2[:0], 'x')
				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
				buf1 = append(buf1, 'x')
				buf2 = append(buf2, 'x')
				want := bytes.EqualFold(buf1, buf2)
				if got := ff.fold(buf1, buf2); got != want {
					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)
				}
			}
		}
	}
}

func MarshalBinary(data []byte, ifc interface{}) error {
	var (
		uix uint64
		ui  uint32
	)

	switch ifc.(type) {
	case byte:
		data[0] = ifc.(byte)
	case float32:
		ui = math.Float32bits(ifc.(float32))
		m32(data, ui)
	case float64:
		uix = math.Float64bits(ifc.(float64))
		m64(data, uix)
	case rune:
		ui = uint32(ifc.(rune))
		m32(data, ui)
	case int:
		ui = uint32(ifc.(int))
		m32(data, ui)
	case uint32:
		ui = ifc.(uint32)
		m32(data, ui)
	case uintptr:
		uix = uint64(ifc.(uintptr))
		for i := uint64(0); i < 8; i++ {
			data[i] = byte((uix >> (8 * i)) & 0xff)
		}
	case uint64:
		uix = ifc.(uint64)
		m64(data, uix)
	case []rune:
		dd := make([]byte, utf8.UTFMax)
		xd := ifc.([]rune)
		n := 0
		for _, r := range xd {
			l := utf8.EncodeRune(dd, r)
			copy(data[n:n+l], dd)
			n += l
		}
	case string:
		dd := make([]byte, utf8.UTFMax)
		xd := ifc.(string)
		n := 0
		for _, r := range xd {
			l := utf8.EncodeRune(dd, r)
			copy(data[n:n+l], dd)
			n += l
		}
	case []byte:
		xd := ifc.([]byte)
		data := make([]byte, len(xd))
		copy(data[0:len(xd)], xd)
	default:
		return errors.New("unknown type for marshal")
	}
	return nil
}

func (b *runeBuilder) addRange(lo, hi rune, fold bool) {
	if lo > hi {
		return
	}

	// TODO: Pick off 80-10FFFF for special handling?
	if lo == 0x80 && hi == 0x10FFFF {
	}

	// Split range into same-length sized ranges.
	for i := 1; i < utf8.UTFMax; i++ {
		max := maxRune(i)
		if lo <= max && max < hi {
			b.addRange(lo, max, fold)
			b.addRange(max+1, hi, fold)
			return
		}
	}

	// ASCII range is special.
	if hi < utf8.RuneSelf {
		b.addBranch(b.suffix(byte(lo), byte(hi), fold, 0))
		return
	}

	// Split range into sections that agree on leading bytes.
	for i := 1; i < utf8.UTFMax; i++ {
		m := rune(1)<<uint(6*i) - 1 // last i bytes of UTF-8 sequence
		if lo&^m != hi&^m {
			if lo&m != 0 {
				b.addRange(lo, lo|m, fold)
				b.addRange((lo|m)+1, hi, fold)
				return
			}
			if hi&m != m {
				b.addRange(lo, hi&^m-1, fold)
				b.addRange(hi&^m, hi, fold)
				return
			}
		}
	}

	// Finally.  Generate byte matching equivalent for lo-hi.
	var ulo, uhi [utf8.UTFMax]byte
	n := utf8.EncodeRune(ulo[:], lo)
	m := utf8.EncodeRune(uhi[:], hi)
	if n != m {
		panic("codesearch/regexp: bad utf-8 math")
	}

	pc := uint32(0)
	for i := n - 1; i >= 0; i-- {
		pc = b.suffix(ulo[i], uhi[i], false, pc)
	}
	b.addBranch(pc)
}

func (f *format) parse() error {

	if len(f.pattern) == 0 {
		return nil
	}
	f.parsed = true // only IF there is a format

	pw, sw := 0, 0 // prefixWritten, suffixWritten
	suffixStart, precStart := false, false
	hasGroup, hasPlus, hasMinus := false, false, false
	precCount := 0
	for _, c := range f.pattern {
		switch c {
		case '+':
			hasPlus = true
		case '-':
			hasMinus = true
		case '#', '0', '.', ',':
			if false == hasGroup && c == ',' {
				hasGroup = true
			}
			if precStart {
				precCount++
			}
			if false == precStart && c == '.' {
				precStart = true
			}
			suffixStart = true
		default:
			if false == suffixStart { // prefix
				if c > 0 {
					pw += utf8.EncodeRune(f.prefix[pw:], c)
					f.prefix = f.prefix[:formatBufferSize]
				}
			} else if c > 0 { // suffix
				sw += utf8.EncodeRune(f.suffix[sw:], c)
				f.suffix = f.suffix[:formatBufferSize]
			}
		}
	}
	f.prefix = f.prefix[:pw]
	f.suffix = f.suffix[:sw]

	if false == hasGroup {
		f.group = 0
	}
	if false == hasPlus {
		f.plusSign = 0
	}
	if false == hasMinus {
		f.minusSign = 0
	}
	f.precision = precCount

	return nil
}

// decomposeHangul writes the decomposed Hangul to buf and returns the number
// of bytes written.  len(buf) should be at least 9.
func decomposeHangul(buf []byte, r rune) int {
	const JamoUTF8Len = 3
	r -= hangulBase
	x := r % jamoTCount
	r /= jamoTCount
	utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
	utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
	if x != 0 {
		utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
		return 3 * JamoUTF8Len
	}
	return 2 * JamoUTF8Len
}

func nextNode(curNode *node, char rune) *node {
	buf := [utf8.UTFMax]byte{}
	for i := 0; i < utf8.EncodeRune(buf[:], char) && curNode != nil; i++ {
		curNode = curNode.getChild(buf[i])
	}
	// Special case: 'q' never occurs by itself, only as 'qu'
	if char == 'q' && curNode != nil {
		for i := 0; i < utf8.EncodeRune(buf[:], 'u') && curNode != nil; i++ {
			curNode = curNode.getChild(buf[i])
		}
	}
	return curNode
}

// Generalized call for multibyte (2 max) decoding (SJIS and CJK). Multibyte to multibyte
func dec_table(in, out chan byte, finished, closed chan error, enc Encoding) {
	table := enc_tables[enc]

	var (
		b     byte
		a     byte
		ok    bool
		multi bool
		sz    int
		cp    = make([]byte, utf8.UTFMax)
	)

loop:
	for {
		select {
		case b, ok = <-in:
			if !ok {
				break loop
			}

			if multi {
				sz = utf8.EncodeRune(cp, table[rune(a)<<8|rune(b)])
				// we don't really care about garbage in the slice
				for _, encoded := range cp[:sz] {
					out <- encoded
				}
			} else {
				if b < 128 {
					utf8.EncodeRune(cp, rune(b))
					out <- cp[0]
				} else {
					multi = true
					a = b
					continue loop
				}
			}
			multi = false

		case <-finished:
			if multi {
				finished <- errors.New("mojibake: dec_table(" + enc.String() + "): malformed byte stream")
			} else {
				finished <- nil
			}
			multi = false

		case <-closed:
			break loop
		}
	}
}

func TestRuneIO(t *testing.T) {
	const NRune = 1000
	// Built a test slice while we write the data
	b := make([]byte, utf8.UTFMax*NRune)
	var buf Buffer
	n := 0
	for r := rune(0); r < NRune; r++ {
		size := utf8.EncodeRune(b[n:], r)
		nbytes, err := buf.WriteRune(r)
		if err != nil {
			t.Fatalf("WriteRune(%U) error: %s", r, err)
		}
		if nbytes != size {
			t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes)
		}
		n += size
	}
	b = b[0:n]

	// Check the resulting bytes
	if !Equal(buf.Bytes(), b) {
		t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b)
	}

	p := make([]byte, utf8.UTFMax)
	// Read it back with ReadRune
	for r := rune(0); r < NRune; r++ {
		size := utf8.EncodeRune(p, r)
		nr, nbytes, err := buf.ReadRune()
		if nr != r || nbytes != size || err != nil {
			t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err)
		}
	}

	// Check that UnreadRune works
	buf.Reset()
	buf.Write(b)
	for r := rune(0); r < NRune; r++ {
		r1, size, _ := buf.ReadRune()
		if err := buf.UnreadRune(); err != nil {
			t.Fatalf("UnreadRune(%U) got error %q", r, err)
		}
		r2, nbytes, err := buf.ReadRune()
		if r1 != r2 || r1 != r || nbytes != size || err != nil {
			t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err)
		}
	}
}