Golang unicode.IsPunct函数代码示例

func find_keywords(dict darts.Darts, line string) map[string]int {
	arr := []rune(strings.ToUpper(line))
	result := make(map[string]int)
	for i := 0; i < len(arr); i++ {
		offset := i
		c := arr[offset]
		if unicode.IsSpace(c) || unicode.IsPunct(c) {
			continue
		}
		for pos := 2; offset+pos < len(arr); pos++ {
			c := arr[offset+pos-1]
			if unicode.IsPunct(c) {
				break
			}
			// log.Info(string(arr[offset : offset+pos]))
			exist, results := dict.CommonPrefixSearch(arr[offset:offset+pos], 0)
			if len(results) > 0 {
				key := string(arr[offset : offset+pos])
				result[key] = result[key] + 1
				offset = offset + pos - 1
			} else if !exist {
				break
			}
		}
	}
	return result
}

func owp(dst io.Writer, src io.Reader) {
	byte_in := func() byte {
		bs := make([]byte, 1)
		src.Read(bs)
		return bs[0]
	}
	byte_out := func(b byte) { dst.Write([]byte{b}) }
	odd := func() byte {
		for {
			b := byte_in()
			if unicode.IsPunct(int(b)) {
				return b
			}
			defer byte_out(b)
		}
		panic("impossible")
	}
	for {
		for {
			b := byte_in()
			byte_out(b)
			if b == '.' {
				return
			}
			if unicode.IsPunct(rune(b)) {
				break
			}
		}
		b := odd()
		byte_out(b)
		if b == '.' {
			return
		}
	}
}

func owp(dst io.Writer, src io.Reader) {
	byte_in := func() byte {
		bs := make([]byte, 1)
		src.Read(bs)
		return bs[0]
	}
	byte_out := func(b byte) { dst.Write([]byte{b}) }
	var odd func() byte
	odd = func() byte {
		s := byte_in()
		if unicode.IsPunct(rune(s)) {
			return s
		}
		b := odd()
		byte_out(s)
		return b
	}
	for {
		for {
			b := byte_in()
			byte_out(b)
			if b == '.' {
				return
			}
			if unicode.IsPunct(rune(b)) {
				break
			}
		}
		b := odd()
		byte_out(b)
		if b == '.' {
			return
		}
	}
}

func DeEscapeProse(p md.Prose) md.Prose {
	result := make(md.Prose, 0, len(p))
	var buf []byte
runs:
	for i := 0; i < len(p); i++ {
		if buf == nil {
			buf = p[i].Bytes
		}
		for j := 0; ; {
			k := bytes.IndexByte(buf[j:], '\\')
			if k == -1 {
				result = append(result, md.Run{
					Line:  p[i].Line,
					Bytes: buf,
				})
				buf = nil
				continue runs
			}
			j += k
			r, _ := utf8.DecodeRune(buf[j+1:])
			if unicode.IsPunct(r) || unicode.IsSymbol(r) {
				result = append(result, md.Run{
					Line:  p[i].Line,
					Bytes: buf[:j],
				})
				buf = buf[j+1:]
				i--
				continue runs
			}
			j++
		}
	}
	return result
}

// Escape escapes the given data to make sure it is safe to use it as a
// filename. It also replaces spaces and other seperation characters
// with the '-' character. It returns an error if the escaped string is
// empty.
func Escape(name string) (escaped string, err error) {
	mfunc := func(r rune) rune {
		switch {
		case unicode.IsLetter(r):
			return r
		case unicode.IsNumber(r):
			return r
		case unicode.IsSpace(r):
			return '-'
		case unicode.IsPunct(r):
			return '-'
		}

		return -1
	}

	escaped = strings.Map(mfunc, html.UnescapeString(name))
	for strings.Contains(escaped, "--") {
		escaped = strings.Replace(escaped, "--", "-", -1)
	}

	escaped = strings.TrimPrefix(escaped, "-")
	escaped = strings.TrimSuffix(escaped, "-")

	if len(escaped) <= 0 {
		err = errors.New("couldn't escape title")
	}

	return
}

func splitText(t string) (ws []string) {
	start := 0
	inWord := false

	for i, r := range t {
		sep := unicode.IsPunct(r) || unicode.IsSpace(r)

		if sep {
			switch {
			case r == '\'': // Accept things like "boy's"

			case inWord:
				ws = append(ws, t[start:i])
				start = i + 1
				inWord = false

			default:
				start += utf8.RuneLen(r)
			}
		}

		inWord = !sep
	}

	if start < len(t) {
		ws = append(ws, t[start:])
	}

	return
}

// IsSearchWordRune defines the runes that can be used in unquoted predicate arguments
// or unquoted literals. These are all unicode letters, digits and punctuation,
// execpt for ':', which is used for predicate marking,  and '(', ')', which are used
// for predicate grouping.
func isSearchWordRune(r rune) bool {
	switch r {
	case ':', ')', '(':
		return false
	}
	return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsPunct(r)
}

// CharCount scans a *bufio.Reader and returns a map of the counts of its
// Unicode character types.
func CharCount(in *bufio.Reader) map[string]int {
	counts := make(map[string]int) // counts of Unicode character types

	for {
		r, n, err := in.ReadRune() // returns rune, nbytes, error
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Fprintf(os.Stderr, "charcount: %v\n", err)
			os.Exit(1)
		}

		switch {
		case r == unicode.ReplacementChar && n == 1:
			counts["invalid"]++
		case unicode.IsControl(r):
			counts["control"]++
		case unicode.IsLetter(r):
			counts["letter"]++
		case unicode.IsMark(r):
			counts["mark"]++
		case unicode.IsNumber(r):
			counts["number"]++
		case unicode.IsPunct(r):
			counts["punct"]++
		case unicode.IsSpace(r):
			counts["space"]++
		case unicode.IsSymbol(r):
			counts["symbol"]++
		}
	}
	return counts
}

// MorseKeys translates an input string into a series of keys.
func MorseKeys(in string) ([]key, error) {
	afterWord := false
	afterChar := false
	result := []key{}
	for _, c := range in {
		if unicode.IsSpace(c) {
			afterWord = true
			continue
		}
		morse, ok := runeToKeys[c]
		if !ok {
			return nil, fmt.Errorf("can't translate %c to morse", c)
		}
		if unicode.IsPunct(c) && afterChar {
			result = append(result, punctGap...)
		} else if afterWord {
			result = append(result, wordGap...)
		} else if afterChar {
			result = append(result, charGap...)
		}
		result = append(result, morse...)
		afterChar = true
		afterWord = false
	}
	return result, nil
}

func main() {
	counts := make(map[rune]int)      // counts of Unicode characters
	var utflen [utf8.UTFMax + 1]int   // count of lengths of UTF-8 encodings
	invalid := 0                      // count of invalid UTF-8 characters
	catCounts := make(map[string]int) // counts per Unicode category
	unknown := 0                      // count of characters of unknown category

	in := bufio.NewReader(os.Stdin)
	for {
		r, n, err := in.ReadRune() // returns rune, nbytes, error
		if err == io.EOF {
			break
		}
		if err != nil {
			fmt.Fprintf(os.Stderr, "charcount: %v\n", err)
			os.Exit(1)
		}
		if r == unicode.ReplacementChar && n == 1 {
			invalid++
			continue
		}
		counts[r]++
		utflen[n]++
		switch {
		case unicode.IsLetter(r):
			catCounts["Letter"]++
		case unicode.IsDigit(r):
			catCounts["Digit"]++
		case unicode.IsSymbol(r):
			catCounts["Symbol"]++
		case unicode.IsPunct(r):
			catCounts["Punct"]++
		case unicode.IsSpace(r):
			catCounts["Space"]++
		default:
			unknown++
		}
	}
	fmt.Printf("rune\tcount\n")
	for c, n := range counts {
		fmt.Printf("%q\t%d\n", c, n)
	}
	fmt.Print("\nlen\tcount\n")
	for i, n := range utflen {
		if i > 0 {
			fmt.Printf("%d\t%d\n", i, n)
		}
	}
	if invalid > 0 {
		fmt.Printf("\n%d invalid UTF-8 characters\n", invalid)
	}

	fmt.Print("\ncat\tcount\n")
	for cat, n := range catCounts {
		fmt.Printf("%s\t%d\n", cat, n)
	}
	if unknown > 0 {
		fmt.Printf("\n%d characters of unknown category\n", unknown)
	}
}

func TestRune_IsIndependent(t *testing.T) {
	numbers := make([]rune, 0)
	letters := make([]rune, 0)
	marks := make([]rune, 0)
	symbols := make([]rune, 0)
	puncts := make([]rune, 0)
	others := make([]rune, 0)
	for _, r := range unicode.Myanmar.R16 {
		for c := r.Lo; c <= r.Hi; c++ {
			switch mr := rune(c); true {
			case unicode.IsLetter(mr):
				letters = append(letters, mr)
			case unicode.IsNumber(mr):
				numbers = append(numbers, mr)
			case unicode.IsMark(mr):
				marks = append(marks, mr)
			case unicode.IsPunct(mr):
				puncts = append(puncts, mr)
			case unicode.IsSymbol(mr):
				symbols = append(symbols, mr)
			default:
				others = append(others, mr)
			}
		}
	}

	independents := string(letters) + string(numbers) + string(puncts) + " \t\r\n"
	for _, consonant := range independents {
		if ok, _ := Rune(consonant).IsIndependent(); !ok {
			t.Errorf("[%U] expected result is true, but it returns false", consonant)
		}
	}
}

// CharType returns a string representing the unicode type of a rune
func CharType(r rune) string {
	switch {
	case unicode.IsLetter(r):
		return "letter"
	case unicode.IsSpace(r):
		return "space"
	case unicode.IsPunct(r):
		return "punct"
	case unicode.IsNumber(r):
		return "number"
	case unicode.IsSymbol(r):
		return "symbol"
	case unicode.IsMark(r):
		return "mark"
	case unicode.IsDigit(r):
		return "digit"
	case unicode.IsPrint(r):
		return "print"
	case unicode.IsControl(r):
		return "control"
	case unicode.IsGraphic(r):
		return "graphic"
	default:
		return "invalid"
	}
}

// Stat calculates statistics for all runes read from r.
func (m *Main) Stat(r io.RuneReader) (Stats, error) {
	var stats Stats

	for {
		// Read next character.
		ch, sz, err := r.ReadRune()
		if err == io.EOF {
			break
		} else if err != nil {
			return stats, err
		}

		// Calculate stats.
		stats.TotalN++
		if unicode.IsControl(ch) {
			stats.ControlN++
		}
		if unicode.IsDigit(ch) {
			stats.DigitN++
		}
		if unicode.IsGraphic(ch) {
			stats.GraphicN++
		}
		if unicode.IsLetter(ch) {
			stats.LetterN++
		}
		if unicode.IsLower(ch) {
			stats.LowerN++
		}
		if unicode.IsMark(ch) {
			stats.MarkN++
		}
		if unicode.IsNumber(ch) {
			stats.NumberN++
		}
		if unicode.IsPrint(ch) {
			stats.PrintN++
		}
		if unicode.IsPunct(ch) {
			stats.PunctN++
		}
		if unicode.IsSpace(ch) {
			stats.SpaceN++
		}
		if unicode.IsSymbol(ch) {
			stats.SymbolN++
		}
		if unicode.IsTitle(ch) {
			stats.TitleN++
		}
		if unicode.IsUpper(ch) {
			stats.UpperN++
		}
		if sz > 1 {
			stats.MultiByteN++
		}
	}

	return stats, nil
}

func incrementCount(r rune, counts map[int]int) {
	switch {
	case unicode.IsControl(r):
		counts[isControl]++

	case unicode.IsNumber(r):
		counts[isNumber]++

	case unicode.IsDigit(r):
		counts[isDigit]++

	case unicode.IsLetter(r):
		counts[isLetter]++

	case unicode.IsMark(r):
		counts[isMark]++

	case unicode.IsPunct(r):
		counts[isPunct]++

	case unicode.IsSpace(r):
		counts[isSpace]++

	case unicode.IsSymbol(r):
		counts[isSymbol]++

	case unicode.IsPrint(r):
		counts[isPrint]++

	case unicode.IsGraphic(r):
		counts[isGraphic]++
	}

}

func test_password(pass string) bool {
	// Windows AD password needs at leat 7 characters password,  and must contain characters from three of the following five categories:
	// uppercase character
	// lowercase character
	// digit character
	// nonalphanumeric characters
	// any Unicode character that is categorized as an alphabetic character but is not uppercase or lowercase
	if len(pass) < 7 {
		return false
	}
	d := 0
	l := 0
	u := 0
	p := 0
	o := 0
	for _, c := range pass {
		if unicode.IsDigit(c) { // check digit character
			d = 1
		} else if unicode.IsLower(c) { // check lowercase character
			l = 1
		} else if unicode.IsUpper(c) { // check uppercase character
			u = 1
		} else if unicode.IsPunct(c) { // check nonalphanumeric character
			p = 1
		} else { // other unicode character
			o = 1
		}
	}
	if d+l+u+p+o < 3 {
		return false
	}
	return true
}