Golang utf16.DecodeRune函数代码示例

func decodeUTF16beRune(p []byte) (r rune, size int, status Status) {
	if len(p) < 2 {
		status = NO_ROOM
		return
	}

	c := rune(p[0])<<8 + rune(p[1])

	if utf16.IsSurrogate(c) {
		if len(p) < 4 {
			status = NO_ROOM
			return
		}

		c2 := rune(p[2])<<8 + rune(p[3])
		c = utf16.DecodeRune(c, c2)

		if c == 0xfffd {
			return c, 2, INVALID_CHAR
		} else {
			return c, 4, SUCCESS
		}
	}

	return c, 2, SUCCESS
}

func unicodeLen(str string) int {
	index := strings.Index(str, `\u`)
	for {
		if index == -1 {
			break
		}

		r1str := str[index+2:index+6] + " " + str[index+8:index+12]

		var r1, r2 rune

		fmt.Sscanf(r1str, "%x %x", &r1, &r2)
		Trace.Println("runes", r1, r2)
		if utf16.IsSurrogate(r1) && utf16.IsSurrogate(r2) {
			r := utf16.DecodeRune(r1, r2)
			if r == replacementChar {
				Warning.Println("Unable to deccode runes")
				str = strings.Replace(str, `\u`, "0x", 2)
			} else {
				str = str[:index] + " " + str[index+12:len(str)]
			}
		}
		index = strings.Index(str, `\u`)
	}
	return len(str)
}

// readConsole reads utf16 characters from console File,
// encodes them into utf8 and stores them in buffer buf.
// It returns the number of utf8 bytes read and an error, if any.
func (f *File) readConsole(buf []byte) (n int, err error) {
	if len(buf) == 0 {
		return 0, nil
	}
	if len(f.readbuf) > 0 {
		return f.copyReadConsoleBuffer(buf)
	}
	wchar, err := f.readOneUTF16FromConsole()
	if err != nil {
		return 0, err
	}
	r := rune(wchar)
	if utf16.IsSurrogate(r) {
		wchar, err := f.readOneUTF16FromConsole()
		if err != nil {
			return 0, err
		}
		r = utf16.DecodeRune(r, rune(wchar))
	}
	if nr := utf8.RuneLen(r); nr > len(buf) {
		start := len(f.readbuf)
		for ; nr > 0; nr-- {
			f.readbuf = append(f.readbuf, 0)
		}
		utf8.EncodeRune(f.readbuf[start:cap(f.readbuf)], r)
	} else {
		utf8.EncodeRune(buf, r)
		buf = buf[nr:]
		n += nr
	}
	if n > 0 {
		return n, nil
	}
	return f.copyReadConsoleBuffer(buf)
}

func (d *jsonDecDriver) appendStringAsBytes() {
	if d.tok == 0 {
		var b byte
		r := d.r
		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
		}
		d.tok = b
	}

	// handle null as a string
	if d.tok == 'n' {
		d.readStrIdx(10, 13) // ull
		d.bs = d.bs[:0]
		return
	}

	if d.tok != '"' {
		d.d.errorf("json: expect char '%c' but got char '%c'", '"', d.tok)
	}
	d.tok = 0

	v := d.bs[:0]
	var c uint8
	r := d.r
	for {
		c = r.readn1()
		if c == '"' {
			break
		} else if c == '\\' {
			c = r.readn1()
			switch c {
			case '"', '\\', '/', '\'':
				v = append(v, c)
			case 'b':
				v = append(v, '\b')
			case 'f':
				v = append(v, '\f')
			case 'n':
				v = append(v, '\n')
			case 'r':
				v = append(v, '\r')
			case 't':
				v = append(v, '\t')
			case 'u':
				rr := d.jsonU4(false)
				// fmt.Printf("$$$$$$$$$: is surrogate: %v\n", utf16.IsSurrogate(rr))
				if utf16.IsSurrogate(rr) {
					rr = utf16.DecodeRune(rr, d.jsonU4(true))
				}
				w2 := utf8.EncodeRune(d.bstr[:], rr)
				v = append(v, d.bstr[:w2]...)
			default:
				d.d.errorf("json: unsupported escaped value: %c", c)
			}
		} else {
			v = append(v, c)
		}
	}
	d.bs = v
}

// utf16toutf8 returns the UTF-8 encoding of the UTF-16 sequence s,
// with a terminating NUL removed.
func utf16toutf8(s []uint16) []byte {
	for i, v := range s {
		if v == 0 {
			s = s[0:i]
			break
		}
	}
	buf := make([]byte, 0, len(s)*2) // allow 2 bytes for every rune
	b := make([]byte, 4)
	for i := 0; i < len(s); i++ {
		var rr rune
		switch r := s[i]; {
		case surr1 <= r && r < surr2 && i+1 < len(s) &&
			surr2 <= s[i+1] && s[i+1] < surr3:
			// valid surrogate sequence
			rr = utf16.DecodeRune(rune(r), rune(s[i+1]))
			i++
		case surr1 <= r && r < surr3:
			// invalid surrogate sequence
			rr = replacementChar
		default:
			// normal rune
			rr = rune(r)
		}
		b := b[:cap(b)]
		n := utf8.EncodeRune(b, rr)
		b = b[:n]
		buf = append(buf, b...)
	}
	return buf
}

func UTF16ToUTF8Bytes(in []byte, out io.Writer) error {
	if len(in)%2 != 0 {
		return fmt.Errorf("input buffer must have an even length (length=%d)", len(in))
	}

	var runeBuf [4]byte
	var v1, v2 uint16
	for i := 0; i < len(in); i += 2 {
		v1 = uint16(in[i]) | uint16(in[i+1])<<8

		switch {
		case v1 < surr1, surr3 <= v1:
			n := utf8.EncodeRune(runeBuf[:], rune(v1))
			out.Write(runeBuf[:n])
		case surr1 <= v1 && v1 < surr2 && len(in) > i+2:
			v2 = uint16(in[i+2]) | uint16(in[i+3])<<8
			if surr2 <= v2 && v2 < surr3 {
				// valid surrogate sequence
				r := utf16.DecodeRune(rune(v1), rune(v2))
				n := utf8.EncodeRune(runeBuf[:], r)
				out.Write(runeBuf[:n])
			}
			i += 2
		default:
			// invalid surrogate sequence
			n := utf8.EncodeRune(runeBuf[:], replacementChar)
			out.Write(runeBuf[:n])
		}
	}

	return nil
}

func (u *utf16Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
	if u.currentBOMPolicy == ExpectBOM {
		if len(src) < 2 {
			return 0, 0, transform.ErrShortSrc
		}
		switch {
		case src[0] == 0xfe && src[1] == 0xff:
			u.endianness = BigEndian
		case src[0] == 0xff && src[1] == 0xfe:
			u.endianness = LittleEndian
		default:
			return 0, 0, ErrMissingBOM
		}
		u.currentBOMPolicy = IgnoreBOM
		nSrc = 2
	}

	var r rune
	var dSize, sSize int
	for nSrc < len(src) {
		if nSrc+1 < len(src) {
			x := uint16(src[nSrc+0])<<8 | uint16(src[nSrc+1])
			if u.endianness == LittleEndian {
				x = x>>8 | x<<8
			}
			r, sSize = rune(x), 2
			if utf16.IsSurrogate(r) {
				if nSrc+3 < len(src) {
					x = uint16(src[nSrc+2])<<8 | uint16(src[nSrc+3])
					if u.endianness == LittleEndian {
						x = x>>8 | x<<8
					}
					// Safe for next iteration if it is not a high surrogate.
					if isHighSurrogate(rune(x)) {
						r, sSize = utf16.DecodeRune(r, rune(x)), 4
					}
				} else if !atEOF {
					err = transform.ErrShortSrc
					break
				}
			}
			if dSize = utf8.RuneLen(r); dSize < 0 {
				r, dSize = utf8.RuneError, 3
			}
		} else if atEOF {
			// Single trailing byte.
			r, dSize, sSize = utf8.RuneError, 3, 1
		} else {
			err = transform.ErrShortSrc
			break
		}
		if nDst+dSize > len(dst) {
			err = transform.ErrShortDst
			break
		}
		nDst += utf8.EncodeRune(dst[nDst:], r)
		nSrc += sSize
	}
	return nDst, nSrc, err
}

func GetKey() (rune, uint16, uint32) {
	code, scan, shift := getKey()
	if code < 0xDC00 || 0xDFFF < code {
		return code, scan, shift
	}
	code2, _, _ := getKey()
	return utf16.DecodeRune(code, code2), scan, shift
}

func (u *utf16Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
	if u.bomPolicy == ExpectBOM {
		if len(src) < 2 {
			return 0, 0, transform.ErrShortSrc
		}
		switch {
		case src[0] == 0xfe && src[1] == 0xff:
			u.endianness = BigEndian
		case src[0] == 0xff && src[1] == 0xfe:
			u.endianness = LittleEndian
		default:
			return 0, 0, ErrMissingBOM
		}
		u.bomPolicy = IgnoreBOM
		nSrc = 2
	}

	for nSrc+1 < len(src) {
		x := uint16(src[nSrc+0])<<8 | uint16(src[nSrc+1])
		if u.endianness == LittleEndian {
			x = x>>8 | x<<8
		}
		r, sSize := rune(x), 2
		if utf16.IsSurrogate(r) {
			if nSrc+3 >= len(src) {
				break
			}
			x = uint16(src[nSrc+2])<<8 | uint16(src[nSrc+3])
			if u.endianness == LittleEndian {
				x = x>>8 | x<<8
			}
			r, sSize = utf16.DecodeRune(r, rune(x)), 4
		}
		dSize := utf8.RuneLen(r)
		if dSize < 0 {
			r, dSize = utf8.RuneError, 3
		}
		if nDst+dSize > len(dst) {
			err = transform.ErrShortDst
			break
		}
		nDst += utf8.EncodeRune(dst[nDst:], r)
		nSrc += sSize
	}

	if err == nil && nSrc != len(src) {
		err = transform.ErrShortSrc
	}
	return nDst, nSrc, err
}

// utf7dec extracts UTF-16-BE bytes from Base64 data and converts them to UTF-8.
// A nil slice is returned if the encoding is invalid.
func utf7dec(b64 []byte) []byte {
	var b []byte

	// Allocate a single block of memory large enough to store the Base64 data
	// (if padding is required), UTF-16-BE bytes, and decoded UTF-8 bytes.
	// Since a 2-byte UTF-16 sequence may expand into a 3-byte UTF-8 sequence,
	// double the space allocation for UTF-8.
	if n := len(b64); b64[n-1] == '=' {
		return nil
	} else if n&3 == 0 {
		b = make([]byte, u7enc.DecodedLen(n)*3)
	} else {
		n += 4 - n&3
		b = make([]byte, n+u7enc.DecodedLen(n)*3)
		copy(b[copy(b, b64):n], []byte("=="))
		b64, b = b[:n], b[n:]
	}

	// Decode Base64 into the first 1/3rd of b
	n, err := u7enc.Decode(b, b64)
	if err != nil || n&1 == 1 {
		return nil
	}

	// Decode UTF-16-BE into the remaining 2/3rds of b
	b, s := b[:n], b[n:]
	j := 0
	for i := 0; i < n; i += 2 {
		r := rune(b[i])<<8 | rune(b[i+1])
		if utf16.IsSurrogate(r) {
			if i += 2; i == n {
				return nil
			}
			r2 := rune(b[i])<<8 | rune(b[i+1])
			if r = utf16.DecodeRune(r, r2); r == uRepl {
				return nil
			}
		} else if u7min <= r && r <= u7max {
			return nil
		}
		j += utf8.EncodeRune(s[j:], r)
	}
	return s[:j]
}

func (d *jsonDecDriver) appendStringAsBytes() {
	d.expectChar('"')
	v := d.bs[:0]
	var c uint8
	for {
		c = d.r.readn1()
		if c == '"' {
			break
		} else if c == '\\' {
			c = d.r.readn1()
			switch c {
			case '"', '\\', '/', '\'':
				v = append(v, c)
			case 'b':
				v = append(v, '\b')
			case 'f':
				v = append(v, '\f')
			case 'n':
				v = append(v, '\n')
			case 'r':
				v = append(v, '\r')
			case 't':
				v = append(v, '\t')
			case 'u':
				rr := d.jsonU4(false)
				// fmt.Printf("$$$$$$$$$: is surrogate: %v\n", utf16.IsSurrogate(rr))
				if utf16.IsSurrogate(rr) {
					rr = utf16.DecodeRune(rr, d.jsonU4(true))
				}
				w2 := utf8.EncodeRune(d.bstr[:], rr)
				v = append(v, d.bstr[:w2]...)
			default:
				d.d.errorf("json: unsupported escaped value: %c", c)
			}
		} else {
			v = append(v, c)
		}
	}
	if jsonTrackSkipWhitespace {
		d.wsSkipped = false
	}
	d.bs = v
}

func bytesToUTF16String(in []byte) string {
	var out bytes.Buffer
	out.WriteString(`u"`)
	for i := 0; i < len(in)/2; i++ {
		u := rune(in[2*i])<<8 | rune(in[2*i+1])
		if utf16.IsSurrogate(u) && i+1 < len(in)/2 {
			u2 := rune(in[2*i+2])<<8 | rune(in[2*i+3])
			r := utf16.DecodeRune(u, u2)
			if r != unicode.ReplacementChar {
				if unicode.IsPrint(r) {
					out.WriteRune(r)
				} else {
					fmt.Fprintf(&out, `\U%08x`, r)
				}
				i++
				continue
			}
		}

		if u == '\n' {
			out.WriteString(`\n`)
		} else if u == '"' {
			out.WriteString(`\"`)
		} else if u == '\\' {
			out.WriteString(`\\`)
		} else if !utf16.IsSurrogate(u) && unicode.IsPrint(u) {
			out.WriteRune(u)
		} else if u <= 0xff {
			fmt.Fprintf(&out, `\x%02x`, u)
		} else {
			fmt.Fprintf(&out, `\u%04x`, u)
		}
	}
	out.WriteString(`"`)

	// Print the trailing byte if needed.
	if len(in)&1 == 1 {
		fmt.Fprintf(&out, " `\\x%02x`", in[len(in)-1])
	}

	return out.String()
}

func getKeys() []keyInfo {
	var numberOfEventsRead uint32
	var events [10]inputRecordT
	var orgConMode uint32

	result := make([]keyInfo, 0, 0)

	getConsoleMode.Call(uintptr(hConin),
		uintptr(unsafe.Pointer(&orgConMode)))
	setConsoleMode.Call(uintptr(hConin), 0)
	var precode rune = 0
	for len(result) <= 0 {
		readConsoleInput.Call(
			uintptr(hConin),
			uintptr(unsafe.Pointer(&events[0])),
			uintptr(len(events)),
			uintptr(unsafe.Pointer(&numberOfEventsRead)))
		for i := uint32(0); i < numberOfEventsRead; i++ {
			if events[i].eventType == KEY_EVENT && events[i].bKeyDown != 0 {
				var keycode = rune(events[i].unicodeChar)
				if keycode != 0 {
					if precode != 0 {
						keycode = utf16.DecodeRune(precode, keycode)
						precode = 0
					} else if utf16.IsSurrogate(keycode) {
						precode = keycode
						continue
					}
				}
				result = append(result, keyInfo{
					keycode,
					events[i].wVirtualKeyCode,
					events[i].dwControlKeyState,
				})
			}
		}
	}
	setConsoleMode.Call(uintptr(hConin), uintptr(orgConMode))
	return result
}

// Decodes UTF-16 surrogate pairs (such as \uD834\uDD1E).
func decodeSurrogates(s string) string {
	var (
		r1  rune = 0
		buf      = new(bytes.Buffer)
	)
	for _, r := range s {
		if utf16.IsSurrogate(r) {
			if r1 == 0 {
				r1 = r
			} else {
				buf.WriteRune(utf16.DecodeRune(r1, r))
				r1 = 0
			}
		} else {
			if r1 != 0 {
				buf.WriteRune(r1)
				r1 = 0
			}
			buf.WriteRune(r)
		}
	}
	return buf.String()
}

func decodeUTF16(input chan byte, output chan rune, BE bool) {
	defer close(output)

	var surrogatePair rune
	for {
		r, ok, e := readNBytes(input, 4, BE)

		if !ok {
			if nil != e {
				panic(e)
			}
			break
		}

		if 0 != surrogatePair {
			if !utf16.IsSurrogate(r) {
				break
				//ошибка поймается в конце
			}
			r = utf16.DecodeRune(surrogatePair, r)

			surrogatePair = 0
		} else {
			if utf16.IsSurrogate(r) {
				surrogatePair = r
				continue
			}
		}

		output <- r
	}

	if 0 != surrogatePair {
		panic("UTF16 ERROR: Не хватает второй половины суррогатной пары\n")
	}
}