Golang unicode.IsLetter函数代码示例

// StringToVersion function parses a string into a Version struct which can be compared
//
// The implementation is based on http://man.he.net/man5/deb-version
// on https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
//
// It uses the dpkg-1.17.25's algorithm  (lib/parsehelp.c)
func StringToVersion(str string) (Version, error) {
	var version Version

	// Trim leading and trailing space
	str = strings.TrimSpace(str)

	if len(str) <= 0 {
		return Version{}, errors.New("Version string is empty")
	}

	// Find Epoch
	sepEpoch := strings.Index(str, ":")
	if sepEpoch > -1 {
		intEpoch, err := strconv.Atoi(str[:sepEpoch])
		if err == nil {
			version.Epoch = intEpoch
		} else {
			return Version{}, errors.New("Epoch in version is not a number")
		}
		if intEpoch < 0 {
			return Version{}, errors.New("Epoch in version is negative")
		}
	} else {
		version.Epoch = 0
	}

	// Find UpstreamVersion / DebianRevision
	sepDebianRevision := strings.LastIndex(str, "-")
	if sepDebianRevision > -1 {
		version.UpstreamVersion = str[sepEpoch+1 : sepDebianRevision]
		version.DebianRevision = str[sepDebianRevision+1:]
	} else {
		version.UpstreamVersion = str[sepEpoch+1:]
		version.DebianRevision = "0"
	}
	// Verify format
	if len(version.UpstreamVersion) == 0 {
		return Version{}, errors.New("No UpstreamVersion in version")
	}

	if !unicode.IsDigit(rune(version.UpstreamVersion[0])) {
		return Version{}, errors.New("UpstreamVersion in version does not start with digit")
	}

	for i := 0; i < len(version.UpstreamVersion); i = i + 1 {
		r := rune(version.UpstreamVersion[i])
		if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(upstreamVersionAllowedSymbols, r) {
			return Version{}, errors.New("invalid character in UpstreamVersion")
		}
	}

	for i := 0; i < len(version.DebianRevision); i = i + 1 {
		r := rune(version.DebianRevision[i])
		if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(debianRevisionAllowedSymbols, r) {
			return Version{}, errors.New("invalid character in DebianRevision")
		}
	}

	return version, nil
}

/*
StringTrimNonAlnum remove non alpha-numeric character at the beginning and end
for `text`.
*/
func StringTrimNonAlnum(text string) string {
	r := []rune(text)
	rlen := len(r)
	start := 0

	for ; start < rlen; start++ {
		if unicode.IsLetter(r[start]) || unicode.IsDigit(r[start]) {
			break
		}
	}

	if start >= rlen {
		return ""
	}

	r = r[start:]
	rlen = len(r)
	end := rlen - 1
	for ; end >= 0; end-- {
		if unicode.IsLetter(r[end]) || unicode.IsDigit(r[end]) {
			break
		}
	}

	if end < 0 {
		return ""
	}

	r = r[:end+1]

	return string(r)
}

func readIdentifier(r *kmgGoReader.Reader) []byte {
	buf := &bytes.Buffer{}
	if r.IsEof() {
		panic(r.GetFileLineInfo() + " unexcept EOF")
	}
	b := r.ReadRune()
	if b == '_' || unicode.IsLetter(b) {
		buf.WriteRune(b)
	} else {
		r.UnreadRune()
		return nil
	}
	for {
		if r.IsEof() {
			return buf.Bytes()
		}
		b := r.ReadRune()
		if b == '_' || unicode.IsLetter(b) || unicode.IsDigit(b) {
			buf.WriteRune(b)
		} else {
			r.UnreadRune()
			return buf.Bytes() // 不是Identifier的东西留个调用者处理
		}
	}
}

func CheckRuneType(last, current rune) index.RuneType {
	if isTermSep(current) {
		return index.TokenSep
	}

	if current > 128 {
		return index.TokenStart
	}

	if unicode.IsLetter(current) {
		if unicode.IsLetter(last) {
			return index.TokenBody
		}
		return index.TokenStart
	}

	if unicode.IsNumber(current) {
		if unicode.IsNumber(last) {
			return index.TokenBody
		}
		return index.TokenStart
	}

	return index.TokenStart
}

// FmtFieldName formats a string as a struct key
//
// Example:
// 	FmtFieldName("foo_id")
// Output: FooID
func FmtFieldName(s string) string {
	runes := []rune(s)
	for len(runes) > 0 && !unicode.IsLetter(runes[0]) && !unicode.IsDigit(runes[0]) {
		runes = runes[1:]
	}
	if len(runes) == 0 {
		return "_"
	}

	s = stringifyFirstChar(string(runes))
	name := lintFieldName(s)
	runes = []rune(name)
	for i, c := range runes {
		ok := unicode.IsLetter(c) || unicode.IsDigit(c)
		if i == 0 {
			ok = unicode.IsLetter(c)
		}
		if !ok {
			runes[i] = '_'
		}
	}
	s = string(runes)
	s = strings.Trim(s, "_")
	if len(s) == 0 {
		return "_"
	}
	return s
}

/**
CSS time unit

@see https://developer.mozilla.org/zh-TW/docs/Web/CSS/time
*/
func lexNumberUnit(l *Lexer) stateFn {
	tok := l.matchKeywordList(ast.UnitTokenList)

	if tok == nil {
		var r = l.next()

		// for an+b syntax
		if r == 'n' && !unicode.IsLetter(l.peek()) {

			l.emit(ast.T_N)

		} else {

			// for other unit tokens
			for unicode.IsLetter(r) {
				r = l.next()
			}
			l.backup()
			if l.length() > 0 {
				l.emit(ast.T_UNIT_OTHERS)
			}

		}
	}

	if l.peek() == ';' {
		return lexStmt
	}
	return lexExpr
}

func validPackageNames(name string) error {
	if len(name) < 1 {
		return fmt.Errorf(errInvalidPackage, "the name can't be blank")
	}

	for i, r := range []rune(name) {
		if i == 0 {
			if !unicode.IsLetter(r) {
				return fmt.Errorf(errInvalidPackage, "the first character must be a letter")
			}
		}

		switch {
		case unicode.IsLetter(r):
		case unicode.IsDigit(r):
		case r == '-':
		case r == '_':
		case r == '.':
			// ok
		default:
			return fmt.Errorf(errInvalidPackage, "all the characters (but the first) must be either letters or digits")
		}
	}

	return nil
}

func lexTypeSelector(l *Lexer) stateFn {
	var r = l.next()
	if !unicode.IsLetter(r) && !IsInterpolationStartToken(r, l.peekBy(2)) {
		l.error("Expecting letter token for tag name selector. got %s", r)
	}

	var foundInterpolation = false
	r = l.next()
	for {
		if IsInterpolationStartToken(r, l.peek()) {
			l.backup()
			lexInterpolation(l, false)
			foundInterpolation = true
		} else if !unicode.IsLetter(r) && !unicode.IsDigit(r) {
			break
		}
		r = l.next()
	}
	l.backup()

	if foundInterpolation {
		l.emit(ast.T_INTERPOLATION_SELECTOR)
	} else {
		l.emit(ast.T_TYPE_SELECTOR)
	}

	return lexSimpleSelector
}

func lexLang(l *Lexer) stateFn {
	/*
		html:lang(fr-ca) { quotes: '« ' ' »' }
		html:lang(de) { quotes: '»' '«' '\2039' '\203A' }
		:lang(fr) > Q { quotes: '« ' ' »' }
		:lang(de) > Q { quotes: '»' '«' '\2039' '\203A' }
	*/
	// [a-z]{2} - [a-z]{2}
	// [a-z]{2}
	var r = l.next()
	if !unicode.IsLetter(r) {
		l.error("Unexpected language token. Got '%s'", r)
	}

	r = l.next()
	if !unicode.IsLetter(r) {
		l.error("Unexpected language token. Got '%s'", r)
	}

	r = l.peek()
	if r == '-' {
		l.next() // skip '-'
		r = l.next()
		if !unicode.IsLetter(r) {
			l.error("Unexpected language token. Got '%s'", r)
		}
		r = l.next()
		if !unicode.IsLetter(r) {
			l.error("Unexpected language token. Got '%s'", r)
		}
	}
	l.emit(ast.T_LANG_CODE)
	return nil
}

func lexIdentifier(l *Lexer) stateFn {
	var r = l.next()
	if !unicode.IsLetter(r) && r != '-' {
		panic("An identifier needs to start with a letter or dash")
	}
	r = l.next()

	for unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' {
		if r == '-' {
			var r2 = l.peek()
			if !unicode.IsLetter(r2) && r2 != '-' {
				l.backup()
				return lexExpr
			}
		}

		r = l.next()
	}
	l.backup()

	if l.peek() == '(' {
		var curTok = l.emit(ast.T_FUNCTION_NAME)

		if curTok.Str == "url" || curTok.Str == "local" {
			lexUrlParam(l)
		} else {
			lexFunctionParams(l)
		}
	} else {
		l.emit(ast.T_IDENT)
	}
	return lexExpr
}

func ConvertToUnderscore(camel string) (string, error) {
	var prevRune rune
	var underscore []rune
	for index, runeChar := range camel {
		if index == 0 {
			if !unicode.IsLetter(runeChar) {
				return "", fmt.Errorf("Table and column names can't start with a character other than a letter.")
			}
			underscore = append(underscore, unicode.ToLower(runeChar))
			prevRune = runeChar
		} else {
			if runeChar == '_' || unicode.IsLetter(runeChar) || unicode.IsDigit(runeChar) {
				//Look for Upper case letters, append _ and make character lower case
				if unicode.IsUpper(runeChar) {
					if !unicode.IsUpper(prevRune) {
						underscore = append(underscore, '_')
					}
					underscore = append(underscore, unicode.ToLower(runeChar))
				} else {
					underscore = append(underscore, runeChar)
				}
			} else {
				return "", fmt.Errorf("Table and column names can't contain non-alphanumeric characters.")
			}
		}
		prevRune = runeChar
	}
	return string(underscore), nil
}

func (s *sc) ident() (sym Sym) {
	assert.For(unicode.IsLetter(s.ch), 20, "character expected")
	var buf []rune
	for {
		buf = append(buf, s.ch)
		s.next()
		if s.err != nil || !(unicode.IsLetter(s.ch) || unicode.IsDigit(s.ch)) {
			break
		}
	}
	if s.err == nil {
		sym.Str = string(buf)
		key := sym.Str
		if s.evil == nil {
			x := true
			s.evil = &x
			if keyTab[key] == None && keyTab[strings.ToUpper(key)] == s.useTab[0] {
				*s.evil = true
			} else if keyTab[key] == s.useTab[0] {
				*s.evil = false
			}
		}
		set := func() {
			if sym.Code = keyTab[key]; sym.Code == None {
				sym.Code = Ident
				sym.User = s.foreignTab[key]
			} else if sym.Code != None {
				ok := false
				for _, u := range s.useTab {
					if u == sym.Code {
						ok = true
						break
					}
				}
				if !ok {
					sym.Code = Ident
					sym.User = s.foreignTab[key]
				}
			}
		}
		if s.evil != nil {
			if *s.evil {
				key = strings.ToUpper(sym.Str)
				if key != sym.Str {
					set()
				} else {
					sym.Code = Ident
				}
			} else {
				set()
			}

		} else {
			sym.Code = Ident
		}
	} else {
		s.mark("error while ident read")
	}
	return
}

//Atbash implements atbash cypher
func Atbash(s string) (cypher string) {
	s = strings.ToLower(s)
	// s = strings.Replace(s, " ", "", -1)
	var code []string
	var block int
	for _, v := range s {
		if unicode.IsLetter(v) || unicode.IsNumber(v) {
			if block == 5 {
				code = append(code, DELIMITER)
				block = 0
			}

			if unicode.IsLetter(v) {
				index := v - 'a'
				cypherIndex := (('z' - 'a') - index)
				code = append(code, string(cypherIndex+'a'))
			} else {
				code = append(code, string(v))

			}

			block++
		}
	}
	cypher = strings.Join(code, "")
	return
}

// our simplified version of MapReduce does not supply a
// key to the Map function, as in the paper; only a value,
// which is a part of the input file content. the return
// value should be a list of key/value pairs, each represented
// by a mapreduce.KeyValue.
func Map(value string) *list.List {
	//content := []string{}
	content := make(map[string]int)
	valueLen := len(value)
	index := 0
	for index < valueLen {
		for index < valueLen && !unicode.IsLetter(rune(value[index])) {
			index++
		}
		start := index
		for index < valueLen && unicode.IsLetter(rune(value[index])) {
			index++
		}
		//		content = append(content, value[start:index])
		content[value[start:index]]++
	}
	l := list.New()
	//	for _, word := range content {
	//		l.PushBack(mapreduce.KeyValue{word, "1"})
	//	}
	for k, v := range content {
		l.PushBack(mapreduce.KeyValue{k, strconv.Itoa(v)})
	}
	return l
}

// extractLetterSequence extracts first word (sequence of letters ending with a non-letter)
// starting with the specified index and wraps it to dateStringLayoutItem according to the type
// of the word.
func extractLetterSequence(originalStr string, index int) (it dateStringLayoutItem) {
	letters := ""

	bytesToParse := []byte(originalStr[index:])
	runeCount := utf8.RuneCount(bytesToParse)

	var isWord bool
	var isDigit bool

	for i := 0; i < runeCount; i++ {
		rune, runeSize := utf8.DecodeRune(bytesToParse)
		bytesToParse = bytesToParse[runeSize:]

		if i == 0 {
			isWord = unicode.IsLetter(rune)
			isDigit = unicode.IsDigit(rune)
		} else {
			if (isWord && (!unicode.IsLetter(rune) && !unicode.IsDigit(rune))) ||
				(isDigit && !unicode.IsDigit(rune)) ||
				(!isWord && unicode.IsLetter(rune)) ||
				(!isDigit && unicode.IsDigit(rune)) {
				break
			}
		}

		letters += string(rune)
	}

	it.item = letters
	it.isWord = isWord
	it.isDigit = isDigit

	return
}