Golang unicode.Is函数代码示例

// Matcher method for iBoundaryCase. If either left or right is not within the
// target string, then -1 should be provided.
func (s *instr) matchBoundaryMode(left rune, right rune) bool {
	if s.mode != iBoundaryCase {
		return false
	}
	switch s.lr {
	case bBeginText:
		return left == -1
	case bBeginLine:
		return left == -1 || left == '\n'
	case bEndText:
		return right == -1
	case bEndLine:
		return right == -1 || right == '\n'
	case bWordBoundary, bNotWordBoundary:
		// TODO: This is ASCII-only at this point.
		word_range := perl_groups['w']
		whitespace_range := perl_groups['s']
		wb := (unicode.Is(word_range, left) && unicode.Is(whitespace_range, right)) || (unicode.Is(whitespace_range, left) && unicode.Is(word_range, right))
		if s.lr == bWordBoundary {
			return wb
		} else {
			return !wb
		}
	}
	panic("unexpected lr mode")
}

// Get name: /first(first|second)*/
// Do not set p.err if the name is missing (unless unexpected EOF is received):
// let the caller provide better context.
func (p *Parser) name() (s string, ok bool) {
	var b byte
	if b, ok = p.mustgetc(); !ok {
		return
	}

	// As a first approximation, we gather the bytes [A-Za-z_:.-\x80-\xFF]*
	if b < utf8.RuneSelf && !isNameByte(b) {
		p.ungetc(b)
		return "", false
	}
	p.buf.Reset()
	p.buf.WriteByte(b)
	for {
		if b, ok = p.mustgetc(); !ok {
			return
		}
		if b < utf8.RuneSelf && !isNameByte(b) {
			p.ungetc(b)
			break
		}
		p.buf.WriteByte(b)
	}

	// Then we check the characters.
	s = p.buf.String()
	for i, c := range s {
		if !unicode.Is(first, c) && (i == 0 || !unicode.Is(second, c)) {
			p.err = p.syntaxError("invalid XML name: " + s)
			return "", false
		}
	}
	return s, true
}

func main() {
	counter := make(map[string]int, 0)

	data := FetchURL(os.Args[1])

	var buffer []rune

	printed := false

	for _, r := range data {
		if unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r) || r == 'ー' {
			buffer = append(buffer, r)

			printed = false
		} else if printed != true {
			printed = true
			counter[string(buffer)] += 1
			buffer = make([]rune, 0)
		}
	}

	for k, v := range counter {
		fmt.Println(v, k)
	}
}

// reversePreservingCombiningCharacters interprets its argument as UTF-8
// and ignores bytes that do not form valid UTF-8.  return value is UTF-8.
func reversePreservingCombiningCharacters(s string) string {
	if s == "" {
		return ""
	}
	p := []rune(s)
	r := make([]rune, len(p))
	start := len(r)
	for i := 0; i < len(p); {
		// quietly skip invalid UTF-8
		if p[i] == utf8.RuneError {
			i++
			continue
		}
		j := i + 1
		for j < len(p) && (unicode.Is(unicode.Mn, p[j]) ||
			unicode.Is(unicode.Me, p[j]) || unicode.Is(unicode.Mc, p[j])) {
			j++
		}
		for k := j - 1; k >= i; k-- {
			start--
			r[start] = p[k]
		}
		i = j
	}
	return (string(r[start:]))
}

func EncCharacter(char int) (bool, string) {

	if unicode.Is(unicode.Cc, char) || unicode.Is(unicode.Cf, char) || unicode.Is(unicode.Co, char) || unicode.Is(unicode.Cs, char) || unicode.Is(unicode.Zl, char) || unicode.Is(unicode.Zp, char) || unicode.Is(unicode.Zs, char) {
		return false, ""
	}
	s := string(char)

	return true, s
}

func JconvCharset(str string) int {
	arr := JconvRune(str)

	// Hiragana test
	is_hiragana := true
	for _, r := range arr {
		if !unicode.Is(unicode.Hiragana, r) {
			is_hiragana = false
			break
		}
	}
	if is_hiragana {
		return 1
	}

	// Katakana test
	is_katakana := true
	for _, r := range arr {
		if !unicode.Is(unicode.Katakana, r) && r != 'ー' {
			is_katakana = false
			break
		}
	}
	if is_katakana {
		return 2
	}

	// Full cjk range
	rt := unicode.RangeTable{
		R16: []unicode.Range16{
			{Lo: 0x3000, Hi: 0x303f, Stride: 1}, // Punctuation
			{Lo: 0x3040, Hi: 0x309f, Stride: 1}, // Hiragana
			{Lo: 0x30a0, Hi: 0x30ff, Stride: 1}, // Katakana
			{Lo: 0x3400, Hi: 0x4dbf, Stride: 1}, // CJK unified ext A
			{Lo: 0x4e00, Hi: 0x9faf, Stride: 1}, // CJK unified
			{Lo: 0xff00, Hi: 0xffef, Stride: 1}, // Romanji and hw-katakana
		},
		R32:         []unicode.Range32{},
		LatinOffset: 0,
	}
	is_cjk := true
	for _, r := range arr {
		if !unicode.Is(&rt, r) {
			is_cjk = false
			break
		}
	}
	if is_cjk {
		return 3
	}

	// Failed to detect charset
	return 0
}

func (m *minificationText) processText(in string) string {
	var buffer bytes.Buffer
	var rRaw, r rune
	var size int
	prevIsSeparator := false
	prevRune := ' '
	isFirst := true
	for len(in) > 0 {
		rRaw, size = utf8.DecodeRuneInString(in)
		r = unicode.ToLower(rRaw)
		isSeparator := !unicode.Is(notSeparatorRT, r)

		// digits
		if isSeparator && !prevIsSeparator {
			rRaw, _ = utf8.DecodeRuneInString(in[size:])
			isSeparator = !m.isDigit(prevRune, r, rRaw)
		}

		if !isSeparator && prevIsSeparator && !isFirst {
			_ = buffer.WriteByte(' ')
		}

		if !isSeparator {
			_, _ = buffer.WriteRune(r)
			isFirst = false
		}

		prevIsSeparator = isSeparator
		prevRune = r
		in = in[size:]
	}

	return buffer.String()
}

func main() {
	fmt.Println(strings.Contains("Hello, world!", "wo"))
	fmt.Println(strings.ContainsAny("Hello, world", "w o"))
	fmt.Println(strings.Count("Hello Helium", "He"))

	s1 := []string{"Hello,", "world"}
	fmt.Println(strings.Join(s1, " "))

	s2 := strings.Split("Hello, world", " ")
	fmt.Println(s2[1])

	s3 := strings.Fields("Hello, world")
	fmt.Println(s3[1])

	f := func(r rune) bool {
		return unicode.Is(unicode.Hangul, r)
	}

	s4 := strings.FieldsFunc("Hello안녕Hello", f)
	fmt.Println(s4)

	fmt.Println(strings.Repeat("Hello", 10))
	fmt.Println(strings.Replace("Hello, world", "world", "go", 1))
	fmt.Println(strings.Replace("Hello Hello", "llo", "Go", 2))
}

func TestPredicate(t *testing.T) {
	testConditional(t, func(rt *unicode.RangeTable, t, f transform.Transformer) transform.Transformer {
		return If(Predicate(func(r rune) bool {
			return unicode.Is(rt, r)
		}), t, f)
	})
}

func lexPackageName(l *Lexer) stateFn {

	// lex package name
	var lastPeriod bool
OUTER:
	for {

		switch r := l.next(); {
		case unicode.IsLetter(r):
			lastPeriod = false
		case r == '.' || r == '_':
			lastPeriod = true
		case unicode.Is(unicode.White_Space, r):
			l.backup()
			break OUTER
		default:
			l.backup()
			lastPeriod = false
			return l.errorf("expected newline after package name")
		}
	}

	if lastPeriod {
		return l.errorf("package names cannot end with a period or underscore")
	}

	// emit package name
	l.emit(TokenPackageName)

	return lexText
}

// Generate a RuneFilter matching a valid Unicode class. If no matching classes
// are found, then this method will return nil.
// Note that if just a single character is given, Categories will be searched
// for this as a prefix (so that 'N' will match 'Nd', 'Nl', 'No' etc).
func matchUnicodeClass(class string) RuneFilter {
	found := false
	match := make([]*unicode.RangeTable, 0)
	if len(class) == 1 {
		// A single character is a shorthand request for any category starting with this.
		for key, r := range unicode.Categories {
			if key[0] == class[0] {
				found = true
				match = append(match, r)
			}
		}
	} else {
		// Search for the unicode class name inside cats/props/scripts.
		options := []map[string]*unicode.RangeTable{
			unicode.Categories, unicode.Properties, unicode.Scripts}
		for _, option := range options {
			if r, ok := option[class]; ok {
				found = true
				match = append(match, r)
			}
		}
	}

	if found {
		return func(r rune) bool {
			for _, table := range match {
				if unicode.Is(table, r) {
					return true
				}
			}
			return false
		}
	}
	return nil
}

func print_rune_is(char rune, props map[string]*unicode.RangeTable) {
	for prop, table := range props {
		if unicode.Is(table, char) {
			fmt.Println("  ", prop)
		}
	}
}

func main() {
	fmt.Println(strings.Index("Hello, world!", "He"))  // 0: He가 맨 처음에 있으므로 0
	fmt.Println(strings.Index("Hello, world!", "wor")) // 7: wor가 8번째에 있으므로 7
	fmt.Println(strings.Index("Hello, world!", "ow"))  // -1: ow는 없으므로 -1

	fmt.Println(strings.IndexAny("Hello, world!", "eo")) // 1: e가 2번째에 있으므로 1
	fmt.Println(strings.IndexAny("Hello, world!", "f"))  // -1: f는 없으므로 -1

	var c byte
	c = 'd'
	fmt.Println(strings.IndexByte("Hello, world!", c)) // 11: d가 12번째에 있으므로 11
	c = 'f'
	fmt.Println(strings.IndexByte("Hello, world!", c)) // -1: f는 없으므로 -1

	var r rune
	r = '언'
	fmt.Println(strings.IndexRune("고 언어", r)) // 4: "언"이 시작되는 인덱스가 4

	f := func(r rune) bool {
		return unicode.Is(unicode.Hangul, r) // r이 한글 유니코드이면 true를 리턴
	}
	fmt.Println(strings.IndexFunc("Go 언어", f))       // 3: 한글이 4번째부터 시작하므로 3
	fmt.Println(strings.IndexFunc("Go Language", f)) // -1: 한글이 없으므로 -1

	fmt.Println(strings.LastIndex("Hello Hello Hello, world!", "Hello"))
	// 12: 마지막 Hello가 13번째에 있으므로 12

	fmt.Println(strings.LastIndexAny("Hello, world", "ol")) // 10: 마지막 l이 11번째에 있으므로 10

	fmt.Println(strings.LastIndexFunc("Go 언어 안녕", f)) // 13: 마지막 한글인 '녕'이 시작되는 인덱스가 13
}

func NormalizeTitle(title string) string {
	normalizedTitle := title
	normalizedTitle = strings.ToLower(normalizedTitle)
	normalizedTitle = RomanizeHepburn(title)
	normalizedTitle = strings.ToLower(normalizedTitle)
	normalizedTitle = RemoveTrailingApostrophe(normalizedTitle)
	normalizedTitle, _, _ = transform.String(transform.Chain(
		norm.NFD,
		transform.RemoveFunc(func(r rune) bool {
			return unicode.Is(unicode.Mn, r)
		}),
		norm.NFC), normalizedTitle)
	normalizedTitle = strings.ToLower(normalizedTitle)
	normalizedTitle = regexp.MustCompile(`\(\d+\)`).ReplaceAllString(normalizedTitle, " ")
	normalizedTitle = strings.Map(func(r rune) rune {
		if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '.' {
			return ' '
		}
		return r
	}, normalizedTitle)
	normalizedTitle = regexp.MustCompile(`\s+`).ReplaceAllString(normalizedTitle, " ")
	normalizedTitle = strings.TrimSpace(normalizedTitle)

	return normalizedTitle
}

func loadSpoolFiles(dirname string, depth int) {
	dh, err := os.Open(dirname)
	o.MightFail(err, "Couldn't open %s", dirname)
	nodes, err := dh.Readdir(-1)
	o.MightFail(err, "Couldn't readdir on %s", dirname)
	if depth > 0 {
		for _, n := range nodes {
			abspath := path.Join(dirname, n.Name())
			if (n.Mode() & os.ModeType) == os.ModeDir {
				// if not a single character, it's not a spool node.
				if len(n.Name()) != 1 {
					continue
				}
				if n.Name() == "." {
					// we're not interested in .
					continue
				}
				nrunes := []rune(n.Name())
				if unicode.Is(unicode.ASCII_Hex_Digit, nrunes[0]) {
					loadSpoolFiles(abspath, depth-1)
				} else {
					o.Warn("Foreign dirent %s found in spool tree", abspath)
				}
			}
		}
	} else {
		// depth == 0 - only interested in files.
		for _, n := range nodes {
			abspath := path.Join(dirname, n.Name())
			if n.Mode()&os.ModeType == 0 {
				if len(n.Name()) != 16 {
					shuffleToCorrupted(abspath, "Filename incorrect length")
					continue
				}
				id, err := strconv.ParseUint(n.Name(), 16, 64)
				if err != nil {
					shuffleToCorrupted(abspath, "Invalid Filename")
					continue
				}
				fh, err := os.Open(abspath)
				if err != nil {
					shuffleToCorrupted(abspath, "Couldn't open")
					continue
				}
				defer fh.Close()
				jr, err := JobRequestFromReader(fh)
				if err != nil || jr.Id != id {
					o.Warn("Couldn't parse?! %s", err)
					shuffleToCorrupted(abspath, "Parse Failure")
					continue
				}
				// Add the request to the registry directly.
				if !RestoreJobState(jr) {
					shuffleToCorrupted(abspath, "Job State Invalid")
				}
			}
		}
	}
}