Golang utf8.RuneLen函数代码示例

// htmlReplacer returns s with runes replaced acccording to replacementTable
// and when badRunes is true, certain bad runes are allowed through unescaped.
func htmlReplacer(s string, replacementTable []string, badRunes bool) string {
	written, b := 0, new(bytes.Buffer)
	for i, r := range s {
		if r < len(replacementTable) {
			if repl := replacementTable[r]; len(repl) != 0 {
				b.WriteString(s[written:i])
				b.WriteString(repl)
				// Valid as long as replacementTable doesn't
				// include anything above 0x7f.
				written = i + utf8.RuneLen(r)
			}
		} else if badRunes {
			// No-op.
			// IE does not allow these ranges in unquoted attrs.
		} else if 0xfdd0 <= r && r <= 0xfdef || 0xfff0 <= r && r <= 0xffff {
			fmt.Fprintf(b, "%s&#x%x;", s[written:i], r)
			written = i + utf8.RuneLen(r)
		}
	}
	if written == 0 {
		return s
	}
	b.WriteString(s[written:])
	return b.String()
}

func runesToString(runes []int) string {
	length := 0
	for _, v := range runes {
		length += utf8.RuneLen(v)
	}
	data := make([]byte, length)
	cur := data
	for _, v := range runes {
		rlen := utf8.RuneLen(v)
		utf8.EncodeRune(cur[0:rlen], v)
		cur = cur[rlen:]
	}
	return string(data)
}

func TestEntityLength(t *testing.T) {
	// We verify that the length of UTF-8 encoding of each value is <= 1 + len(key).
	// The +1 comes from the leading "&". This property implies that the length of
	// unescaped text is <= the length of escaped text.
	for k, v := range entity {
		if 1+len(k) < utf8.RuneLen(v) {
			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v))
		}
	}
	for k, v := range entity2 {
		if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) {
			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1]))
		}
	}
}

// Map returns a copy of the byte array s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement.  The characters in s and the
// output are interpreted as UTF-8-encoded Unicode code points.
func Map(mapping func(rune int) int, s []byte) []byte {
	// In the worst case, the array can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for i := 0; i < len(s); {
		wid := 1
		rune := int(s[i])
		if rune >= utf8.RuneSelf {
			rune, wid = utf8.DecodeRune(s[i:])
		}
		rune = mapping(rune)
		if rune >= 0 {
			if nbytes+utf8.RuneLen(rune) > maxbytes {
				// Grow the buffer.
				maxbytes = maxbytes*2 + utf8.UTFMax
				nb := make([]byte, maxbytes)
				copy(nb, b[0:nbytes])
				b = nb
			}
			nbytes += utf8.EncodeRune(b[nbytes:maxbytes], rune)
		}
		i += wid
	}
	return b[0:nbytes]
}

// replace replaces each rune r of s with replacementTable[r], provided that
// r < len(replacementTable). If replacementTable[r] is the empty string then
// no replacement is made.
// It also replaces runes U+2028 and U+2029 with the raw strings `\u2028` and
// `\u2029`.
func replace(s string, replacementTable []string) string {
	var b bytes.Buffer
	written := 0
	for i, r := range s {
		var repl string
		switch {
		case int(r) < len(replacementTable) && replacementTable[r] != "":
			repl = replacementTable[r]
		case r == '\u2028':
			repl = `\u2028`
		case r == '\u2029':
			repl = `\u2029`
		default:
			continue
		}
		b.WriteString(s[written:i])
		b.WriteString(repl)
		written = i + utf8.RuneLen(r)
	}
	if written == 0 {
		return s
	}
	b.WriteString(s[written:])
	return b.String()
}

func Write(fd int, p []byte) (n int, errno int) {
	var mode uint32
	var done uint32
	if isConsole, _ := GetConsoleMode(int32(fd), &mode); UnicodeConsoleOutput && isConsole {
		// TODO: The number of TCHARs to write. If the total size of the
		// specified number of characters exceeds 64 KB, the function fails with ERROR_NOT_ENOUGH_MEMORY.
		buf16 := utf16.Encode([]int(string(p)))
		//for _, c := range buf16 { print(c," ") } ; println()
		if ok, e := WriteConsole(int32(fd), buf16, &done); !ok {
			return 0, e
		}
		// convert length of utf16 characters to number of bytes written
		if done == uint32(len(buf16)) {
			done = uint32(len(p))
		} else {
			done = 0
			for _, rune := range utf16.Decode(buf16[:done]) {
				done += uint32(utf8.RuneLen(rune))
			}
		}
	} else {
		// TODO: This might as well fail with large writes, only Microsoft doesn't say that, see
		// http://code.google.com/p/msysgit/issues/detail?id=409 for example
		if ok, e := syscall.WriteFile(int32(fd), p, &done, nil); !ok {
			return 0, e
		}
	}
	return int(done), 0
}

// Map returns a copy of the string s with all its characters modified
// according to the mapping function.
func Map(mapping func(rune int) int, s string) string {
	// In the worst case, the string can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	b := make([]byte, maxbytes)
	for _, c := range s {
		rune := mapping(c)
		wid := 1
		if rune >= utf8.RuneSelf {
			wid = utf8.RuneLen(rune)
		}
		if nbytes+wid > maxbytes {
			// Grow the buffer.
			maxbytes = maxbytes*2 + utf8.UTFMax
			nb := make([]byte, maxbytes)
			for i, c := range b[0:nbytes] {
				nb[i] = c
			}
			b = nb
		}
		nbytes += utf8.EncodeRune(rune, b[nbytes:maxbytes])
	}
	return string(b[0:nbytes])
}

func TestEntityLength(t *testing.T) {
	// We verify that the length of UTF-8 encoding of each value is <= 1 + len(key).
	// The +1 comes from the leading "&". This property implies that the length of
	// unescaped text is <= the length of escaped text.
	for k, v := range entity {
		if 1+len(k) < utf8.RuneLen(v) {
			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v))
		}
		if len(k) > longestEntityWithoutSemicolon && k[len(k)-1] != ';' {
			t.Errorf("entity name %s is %d characters, but longestEntityWithoutSemicolon=%d", k, len(k), longestEntityWithoutSemicolon)
		}
	}
	for k, v := range entity2 {
		if 1+len(k) < utf8.RuneLen(v[0])+utf8.RuneLen(v[1]) {
			t.Error("escaped entity &" + k + " is shorter than its UTF-8 encoding " + string(v[0]) + string(v[1]))
		}
	}
}

func main() {
	fmt.Println("Strings now...")
	for i := 1; i <= 100; i++ {
		for j := 0; j <= i; j++ {
			fmt.Printf("A")
		}
		fmt.Println()
	}

	fmt.Println("Counting chars")
	input := "asSASA ddd dsjkdsjs dk"
	fmt.Printf("Number of chars in %s: %d\n", input, len(input))
	bytes := []byte(input)
	fmt.Printf("Number of bytes in %s: %d\n", input, len(bytes))

	fmt.Println("Now with unicode...")
	input = "aΦx"
	for pos, char := range input {
		fmt.Printf("character '%c' starts at byte position %d\n", char, pos)
	}

	var num_bytes = 0
	for _, char := range input {
		num_bytes += utf8.RuneLen(char)
	}
	fmt.Printf("%s contains %d bytes\n", input, num_bytes)

	fmt.Println("Swapping chars...")
	input = "asSASA ddd dsjkdsjs dk"

	var output = ""
	for pos, char := range input {
		switch pos {
		case 3:
			output = output + "a"
		case 4:
			output = output + "b"
		case 5:
			output = output + "c"
		default:
			output = output + string(char)
		}
	}

	fmt.Println(output)

	fmt.Println("Swapping chars...")
	input = "foobar"
	temp := []byte(input)

	for i, j := 0, len(temp)-1; i < j; i, j = i+1, j-1 {
		temp[i], temp[j] = temp[j], temp[i]
	}

	fmt.Println(string(temp))
}

func urlquoter(c int, safe string) []byte {
	safe_bytes := strings.Bytes(safe);
	c_bytes := make([]byte, utf8.RuneLen(c));
	utf8.EncodeRune(c, c_bytes);
	if bytes.Index(safe_bytes, c_bytes) != -1 || bytes.Index(always_safe, c_bytes) != -1 {
		return c_bytes;
	}
	else {
		return strings.Bytes(fmt.Sprintf("%%%02X", c));
	}
	panic("unreachable");
}

/*
Capitalizes the first character of the value.

Example:

	{value|capfirst}

If value is "neste", the output will be "Neste".
*/
func CapFirstFormatter(w io.Writer, formatter string, data ...interface{}) {
	b := getBytes(data...)

	if len(b) > 0 {
		rune, size := utf8.DecodeRune(b)
		rune = unicode.ToUpper(rune)
		capSize := utf8.RuneLen(rune)
		capb := make([]byte, capSize)
		utf8.EncodeRune(capb, rune)
		w.Write(capb)
		w.Write(b[size:])
	}
}

func (r *DelimReader) Read(p []byte) (n int, err os.Error) {
	bytes_written := 0

	if r.remainder != nil {
		for i := 0; i < len(r.remainder); i++ {
			p[i] = r.remainder[i]
			bytes_written++
		}
		r.remainder = nil
	}

	for bytes_written < len(p) {
		rune, size, err := r.reader.ReadRune()
		if err != nil {
			return bytes_written, err
		}
		for _, value := range r.delimiters {
			if value == rune {
				rune = r.used_delimiter
				size = utf8.RuneLen(rune)
			}
		}
		if bytes_written+size > len(p) {
			// we need to split the rune and hold on to the remainder
			writable := len(p) - bytes_written
			target := make([]byte, size)
			_ = utf8.EncodeRune(target, rune)
			for i := 0; i < writable; i++ {
				p[bytes_written] = target[i]
				bytes_written++
			}
			r.remainder = target[writable:]
		} else {
			target := p[bytes_written : bytes_written+size]
			_ = utf8.EncodeRune(target, rune)
			bytes_written += size
		}
	}
	return bytes_written, nil
}

// Return all anchored substrings of the given string within the Trie, with a matching set of
// their associated values.
func (p *Trie) AllSubstringsAndValues(s string) (*vector.StringVector, *vector.Vector) {
	sv := new(vector.StringVector)
	vv := new(vector.Vector)

	for pos, rune := range s {
		child, ok := p.children[rune]
		if !ok {
			// return whatever we have so far
			break
		}

		// if this is a leaf node, add the string so far and its value
		if child.leaf {
			sv.Push(s[0 : pos+utf8.RuneLen(rune)])
			vv.Push(child.value)
		}

		p = child
	}

	return sv, vv
}

// Map returns a copy of the string s with all its characters modified
// according to the mapping function. If mapping returns a negative value, the character is
// dropped from the string with no replacement.
func Map(mapping func(rune) rune, s string) string {
	// In the worst case, the string can grow when mapped, making
	// things unpleasant.  But it's so rare we barge in assuming it's
	// fine.  It could also shrink but that falls out naturally.
	maxbytes := len(s) // length of b
	nbytes := 0        // number of bytes encoded in b
	// The output buffer b is initialized on demand, the first
	// time a character differs.
	var b []byte

	for i, c := range s {
		r := mapping(c)
		if b == nil {
			if r == c {
				continue
			}
			b = make([]byte, maxbytes)
			nbytes = copy(b, s[:i])
		}
		if r >= 0 {
			wid := 1
			if r >= utf8.RuneSelf {
				wid = utf8.RuneLen(r)
			}
			if nbytes+wid > maxbytes {
				// Grow the buffer.
				maxbytes = maxbytes*2 + utf8.UTFMax
				nb := make([]byte, maxbytes)
				copy(nb, b[0:nbytes])
				b = nb
			}
			nbytes += utf8.EncodeRune(b[nbytes:maxbytes], r)
		}
	}
	if b == nil {
		return s
	}
	return string(b[0:nbytes])
}

func tr(conn *irc.Conn, nick *irc.Nick, args, target string) {
	if args == "" {
		return
	}

	var sourcelang, targetlang, text string
	index := strings.IndexAny(args, " 　") // handle spaces and ideographic spaces (U+3000)
	if index == 5 && args[2] == '|' {
		sourcelang = args[:2]
		targetlang = args[3:5]
		if args[5] == ' ' {
			text = args[6:]
		} else {
			text = args[5+utf8.RuneLen(3000):]
		}
	} else {
		sourcelang = "auto"
		targetlang = "en"
		text = args
	}

	say(conn, target, translate(sourcelang, targetlang, text))
}