Golang utf8.DecodeRuneInString函数代码示例

func IndexFold(s, sub string, offset int) (int, int) {
	sub = strings.ToLower(sub)
	n := len(sub)
	if n == 0 {
		return 0, 0
	}
	sc, _ := utf8.DecodeRuneInString(sub)
	for i := offset; i+n <= len(s); {
		c, size := utf8.DecodeRuneInString(s[i:])
		if unicode.ToLower(c) == sc {
			fail := false
			var i_, j int
			for i_, j = i, 0; j < n; {
				c, size := utf8.DecodeRuneInString(s[i_:])
				c2, size2 := utf8.DecodeRuneInString(sub[j:])
				if unicode.ToLower(c) != c2 {
					fail = true
					break
				}
				i_ += size
				j += size2
			}
			if !fail {
				return i, i_
			}
		}
		i += size
	}
	return -1, -1
}

func (m *minificationText) processText(in string) string {
	var buffer bytes.Buffer
	var rRaw, r rune
	var size int
	prevIsSeparator := false
	prevRune := ' '
	isFirst := true
	for len(in) > 0 {
		rRaw, size = utf8.DecodeRuneInString(in)
		r = unicode.ToLower(rRaw)
		isSeparator := !unicode.Is(notSeparatorRT, r)

		// digits
		if isSeparator && !prevIsSeparator {
			rRaw, _ = utf8.DecodeRuneInString(in[size:])
			isSeparator = !m.isDigit(prevRune, r, rRaw)
		}

		if !isSeparator && prevIsSeparator && !isFirst {
			_ = buffer.WriteByte(' ')
		}

		if !isSeparator {
			_, _ = buffer.WriteRune(r)
			isFirst = false
		}

		prevIsSeparator = isSeparator
		prevRune = r
		in = in[size:]
	}

	return buffer.String()
}

// check if the input is a valid symbol.
// a symbol is anything that starts with a letter or _
// and don't have any whitespace between
// this-is-a-valid-symbol
// this!is_another?crazy_symbol
// THIS
func symbol(in string) (string, string, bool) {
	orig := in
	sym, sz := utf8.DecodeRuneInString(in)
	if isSpace(sym) || isDigit(sym) {
		// a sym MUST START with something different
		// from a digit or space
		return "", orig, false
	}
	in = in[sz:]
	// okay, go ahread and read everything until you find
	// a space
	for len(in) > 0 {
		r, w := utf8.DecodeRuneInString(in)
		if isSpace(r) {
			break
		} else {
			// not a whitespace
			// move the sz counter by w bytes
			// and use the tail of input
			in = in[w:]
			sz += w
		}
	}
	// the first space found is kept intact
	return orig[0:sz], in, true
}

func defaultFileName(lang string, pkg *types.Package) string {
	switch lang {
	case "java":
		if pkg == nil {
			return "Universe.java"
		}
		firstRune, size := utf8.DecodeRuneInString(pkg.Name())
		className := string(unicode.ToUpper(firstRune)) + pkg.Name()[size:]
		return className + ".java"
	case "go":
		if pkg == nil {
			return "go_universe.go"
		}
		return "go_" + pkg.Name() + ".go"
	case "objc":
		if pkg == nil {
			return "GoUniverse.m"
		}
		firstRune, size := utf8.DecodeRuneInString(pkg.Name())
		className := string(unicode.ToUpper(firstRune)) + pkg.Name()[size:]
		return "Go" + className + ".m"
	}
	errorf("unknown target language: %q", lang)
	os.Exit(exitStatus)
	return ""
}

func (l *Lexer) peekNextRune() (rune, int) {
	if l.left() >= 5 {
		return utf8.DecodeRuneInString(l.peek(5))
	} else {
		return utf8.DecodeRuneInString(l.remainder())
	}
}

// 替换文字、参数为要替换的文字内容
func (self *StringFilter) Replace(txt string) string {
	if len(txt) < 1 {
		return txt
	}

	node := self.root

	key := []rune(txt)
	var chars []rune = nil
	slen := len(key)

	for i := 0; i < slen; i++ {
		var match bool
		var endPos int
		if _, exists := node.children[key[i]]; exists {
			node = node.children[key[i]]
			if node.end { // 单个单词匹配
				c, _ := utf8.DecodeRuneInString("*")
				if chars == nil {
					chars = key
				}
				chars[i] = c
			}
			for j := i + 1; j < slen; j++ {
				if _, exists := node.children[key[j]]; !exists {
					break
				}

				node = node.children[key[j]]
				if !node.end {
					continue
				}

				match = true
				endPos = j

				if len(node.children) > 0 {
					continue
				}
			}

			if match {
				if chars == nil {
					chars = key
				}
				for t := i; t <= endPos; t++ { // 从敏感词开始到结束依次替换为*
					c, _ := utf8.DecodeRuneInString("*")
					chars[t] = c
				}

			}
			node = self.root
		}
	}
	if chars == nil {
		return txt
	} else {
		return string(chars)
	}
}

func AreOneEditAway(input1, input2 string) bool {
	len1 := utf8.RuneCountInString(input1)
	len2 := utf8.RuneCountInString(input2)
	if len1 != len2 && len1-1 != len2 && len2-1 != len1 {
		return false
	}
	if len1 == len2 { // must be one replacement
		var width1, width2 int
		var r1, r2 rune
		diffSeen := false
		for i, j := 0, 0; i < len1 || j < len2; i, j = i+width1, j+width2 {
			r1, width1 = utf8.DecodeRuneInString(input1[i:])
			r2, width2 = utf8.DecodeRuneInString(input2[j:])
			if r1 != r2 {
				if diffSeen {
					return false
				} else {
					diffSeen = true
				}
			}
		}
		return true
	} else if len1-1 == len2 { // input1 must be a removal from input2
		return oneRemovalAway(input2, input1)
	} else { //if len2-1 == len1 { // input2 must be a removal from input1
		return oneRemovalAway(input1, input2)
	}
}

func matchIgnoreCase1(s string) (end int) {
	end = -1
	var r rune
	var rlen int
	i := 0
	_, _, _ = r, rlen, i
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		return
	}
	i += rlen
	switch {
	case r == 65 || r == 97:
		goto s2
	}
	return
s2:
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		return
	}
	i += rlen
	switch {
	case r == 90 || r == 122:
		end = i
	}
	return
}

// Evaluate an attribute value template
func evalAVT(input string, node xml.Node, context *ExecutionContext) (out string) {
	var start, pos int
	var inSQlit, inDQlit bool
	for pos < len(input) {
		r, width := utf8.DecodeRuneInString(input[pos:])
		pos += width
		if r == '\'' {
			inSQlit = !inSQlit
		}
		if r == '"' {
			inDQlit = !inDQlit
		}
		if r == '{' {
			// if we're not the last character
			if pos < len(input) {
				// check for doubled opening brace
				peek, w := utf8.DecodeRuneInString(input[pos:])
				if peek == '{' {
					out = out + input[start:pos]
					pos += w
					start = pos
					continue
				}
			}
			out = out + input[start:pos-width]
			start = pos
		}
		if r == '}' {
			if inSQlit || inDQlit {
				continue
			}
			// if we're not the last character
			if pos < len(input) {
				// check for doubled closing brace
				peek, w := utf8.DecodeRuneInString(input[pos:])
				if peek == '}' {
					out = out + input[start:pos]
					pos += w
					start = pos
					continue
				}
			}
			expr := input[start : pos-width]
			ret, _ := context.EvalXPath(node, expr)
			switch val := ret.(type) {
			case []xml.Node:
				for _, n := range val {
					out = out + n.Content()
				}
			case float64:
				out = out + fmt.Sprintf("%v", val)
			case string:
				out = out + val
			}
			start = pos
		}
	}
	out = out + input[start:pos]
	return
}

func matchLazy6(s string) (end int) {
	end = -1
	var r rune
	var rlen int
	i := 0
	lazy := false
	type jmp struct{ s, i int }
	var lazyArr [1]jmp
	lazyStack := lazyArr[:0]
	_, _, _ = r, rlen, i
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		goto bt
	}
	i += rlen
	switch {
	case r == 97:
		goto s2
	}
	goto bt
s2:
	if lazy {
		lazy = false
		goto s3
	}
	lazyStack = append(lazyStack, jmp{s: 2, i: i})
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		goto bt
	}
	i += rlen
	switch {
	case r == 98:
		end = i
	}
	goto bt
s3:
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		goto bt
	}
	i += rlen
	switch {
	case r == 97:
		goto s2
	}
bt:
	if end >= 0 || len(lazyStack) == 0 {
		return
	}
	var to jmp
	to, lazyStack = lazyStack[len(lazyStack)-1], lazyStack[:len(lazyStack)-1]
	lazy = true
	i = to.i
	switch to.s {
	case 2:
		goto s2
	}
	return
}

func (n NeologdNormalizer) EliminateSpace(s string) string {
	var (
		b    bytes.Buffer
		prev rune
	)
	for p := 0; p < len(s); {
		c, w := utf8.DecodeRuneInString(s[p:])
		p += w
		if !unicode.IsSpace(c) {
			b.WriteRune(c)
			prev = c
			continue
		}
		for p < len(s) {
			c0, w0 := utf8.DecodeRuneInString(s[p:])
			p += w0
			if !unicode.IsSpace(c0) {
				if unicode.In(prev, unicode.Latin, latinSymbols) &&
					unicode.In(c0, unicode.Latin, latinSymbols) {
					b.WriteRune(' ')
				}
				b.WriteRune(c0)
				prev = c0
				break
			}
		}

	}
	return b.String()
}

func (this *JCConv) Init() {
	// Hiragana
	hira_txt := "が ぎ ぐ げ ご ざ じ ず ぜ ぞ だ ぢ づ で ど ば び ぶ べ ぼ ぱ ぴ ぷ ぺ ぽ " +
		"あ い う え お か き く け こ さ し す せ そ た ち つ て と " +
		"な に ぬ ね の は ひ ふ へ ほ ま み む め も や ゆ よ ら り る れ ろ " +
		"わ を ん ぁ ぃ ぅ ぇ ぉ ゃ ゅ ょ っ"
	hira_arr := strings.Split(hira_txt, " ")
	this.hira = map[rune]bool{}
	for _, ch := range hira_arr {
		r, _ := utf8.DecodeRuneInString(ch)
		this.hira[r] = true
	}

	// Katakana
	kata_txt := "ガ ギ グ ゲ ゴ ザ ジ ズ ゼ ゾ ダ ヂ ヅ デ ド バ ビ ブ ベ ボ パ ピ プ ペ ポ " +
		"ア イ ウ エ オ カ キ ク ケ コ サ シ ス セ ソ タ チ ツ テ ト " +
		"ナ ニ ヌ ネ ノ ハ ヒ フ ヘ ホ マ ミ ム メ モ ヤ ユ ヨ ラ リ ル レ ロ " +
		"ワ ヲ ン ァ ィ ゥ ェ ォ ャ ュ ョ ッ"
	kata_arr := strings.Split(kata_txt, " ")
	this.kata = map[rune]bool{}
	for _, ch := range kata_arr {
		r, _ := utf8.DecodeRuneInString(ch)
		this.kata[r] = true
	}

	// Conversion map
	this.kata2hira = map[rune]rune{}
	for i, kata := range kata_arr {
		kata_r, _ := utf8.DecodeRuneInString(kata)
		hira_r, _ := utf8.DecodeRuneInString(hira_arr[i])
		this.kata2hira[kata_r] = hira_r
	}
}

func (b *Buffer) TranslationRegex(m Message) Message {
	translation := make(map[rune]rune)
	str1, str2 := splitRegex(m.content)
	for len(str1) > 0 && len(str2) > 0 {
		r1, size1 := utf8.DecodeRuneInString(str1)
		r2, size2 := utf8.DecodeRuneInString(str2)
		translation[r1] = r2
		str1 = str1[size1:]
		str2 = str2[size2:]
	}
	if len(str1) != len(str2) {
		return Message{m.nick, "Translations have different lengths"}
	}
	pos, msg := b.TranslationFindMatch(translation)
	if pos == -1 {
		return Message{m.nick, "No match found."}
	}
	output := make([]rune, 0, len(msg.content))
	for len(msg.content) > 0 {
		r, size := utf8.DecodeRuneInString(msg.content)
		if t, ok := translation[r]; ok {
			output = append(output, t)
		} else {
			output = append(output, r)
		}
		msg.content = msg.content[size:]
	}
	b.messages = append(b.messages[:pos], b.messages[pos+1:]...)
	b.prepend(Message{msg.nick, string(output)})
	return Message{msg.nick, string(output)}
}

func main() {
	f, err := ioutil.ReadFile("./japanese.txt")
	m := map[string]string{}
	if err == nil {
		allString := string(f)
		restpart := allString
		for strings.Index(restpart, "(") != -1 {
			word1_end_idx := strings.Index(restpart, "(")
			beginpart := restpart[0:word1_end_idx]
			restpart = restpart[word1_end_idx+1:]
			word1_begin_idx := strings.LastIndex(beginpart, " ")
			key := beginpart[word1_begin_idx+1 : word1_end_idx]
			roma1_end_idx := strings.Index(restpart, ")")
			val := restpart[:roma1_end_idx]
			m[key] = val
		}
	}

	f, err = ioutil.ReadFile("./source.txt")
	if err == nil {
		allString := string(f)
		restpart := strings.TrimSpace(allString)
		var output, announce string
		for len(restpart) > 0 {
			_, length1 := utf8.DecodeRuneInString(restpart)
			_, length2 := utf8.DecodeRuneInString(restpart[length1:])
			length2 += length1
			fmt.Println("length1:", length1)
			fmt.Println("length2:", length2)
			if m[restpart[:length2]] != "" {
				output = strings.Join([]string{output, restpart[:length2], m[restpart[:length2]]}, "")
				announce = strings.Join([]string{announce, m[restpart[:length2]]}, " ")
				restpart = restpart[length2:]
			} else if m[restpart[:length1]] != "" {
				output = strings.Join([]string{output, restpart[:length1], m[restpart[:length1]]}, "")
				announce = strings.Join([]string{announce, m[restpart[:length1]]}, " ")
				restpart = restpart[length1:]
			} else if restpart[:length1] == "\n" {
				//fmt.Println(1111)
				announce = strings.Join([]string{announce, "\r\n"}, "")
				restpart = restpart[length1:]
			} else if restpart[:length1] == " " {
				announce = strings.Join([]string{announce, " "}, "")
				output = output + " "
				restpart = restpart[length1:]
			} else {
				output = strings.Join([]string{output, restpart[:length1]}, "")
				engletterreg := regexp.MustCompile("[a-zA-Z0-9]")
				announce = strings.Join([]string{announce, engletterreg.FindString(restpart[:length1])}, "")
				restpart = restpart[length1:]
			}
		}
		// for key, val := range m {
		// 	fmt.Println(1, key, 2, val)
		// }
		fmt.Println(m)
		output = strings.Join([]string{output, announce}, "\r\n")
		ioutil.WriteFile("./output.txt", []byte(output), os.ModeAppend)
	}
}

func matchPlus(s string) (end int) {
	end = -1
	var r rune
	var rlen int
	i := 0
	_, _, _ = r, rlen, i
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		return
	}
	i += rlen
	switch {
	case r == 97:
		end = i
		goto s2
	}
	return
s2:
	r, rlen = utf8.DecodeRuneInString(s[i:])
	if rlen == 0 {
		return
	}
	i += rlen
	switch {
	case r == 97:
		end = i
		goto s2
	}
	return
}