Golang Lexer.Next方法代码示例

func LexForExpr(l *lex.Lexer) lex.StateFn {
	if int(l.Pos) >= len(l.Input) {
		l.Emit(lex.ItemEOF)
		return nil
	}
	if strings.HasPrefix(l.Input[l.Pos:], "in") {
		l.Pos += lex.Pos(2)
		l.Emit(ItemIn)
		return InsideAction
	}

	// any list of space separated identifiers
	switch r := l.Next(); {
	case lex.IsSpace(r):
		for lex.IsSpace(l.Peek()) {
			l.Next()
		}
		l.Emit(ItemSpace)
		return LexForExpr
	case unicode.IsLetter(r):
		return LoopIdentifier
	default:
		return l.Errorf("Unexpected Character '%s'", string(r))
	}
}

func Identifier(l *lex.Lexer) lex.StateFn {
	typ := ItemIdentifier
	r, _ := utf8.DecodeRuneInString(l.Input[l.Start:])
	if r == '!' {
		l.Ignore()
		typ = ItemBuiltin
	}
	for {
		// should not appear in the first identifier of an expression of sub-expression (since it must be a function)
		r = l.Next()

		if r == ':' {
			typ = ItemArgKey
			l.Backup()
			l.Emit(typ)
			l.Next()
			l.Ignore()
			return InsideAction
		}
		if !IsValidIdentifierChar(r) {
			l.Backup()
			break
		}
	}
	l.Emit(typ)
	return InsideAction
}

func Space(l *lex.Lexer) lex.StateFn {
	for lex.IsSpace(l.Peek()) {
		l.Next()
	}
	l.Emit(ItemSpace)

	return InsideAction
}

func LoopIdentifier(l *lex.Lexer) lex.StateFn {
	typ := ItemIdentifier
	for {
		r := l.Next()
		if !IsValidIdentifierChar(r) {
			l.Backup()
			break
		}
	}
	l.Emit(typ)
	return LexForExpr
}

func InsideAction(l *lex.Lexer) lex.StateFn {

	if l.Name == "LexTextNode" && strings.HasPrefix(l.Input[l.Pos:], rightDelim) {
		if l.IntState[ActionParenDepth] > 0 {
			return l.Errorf("unmatched parentheses")
		}
		return RightDelim
	}

	switch r := l.Next(); {
	case (r == lex.EOF || lex.IsEndOfLine(r)):
		if l.Name == "LexIfExpr" {
			return LexIfExpr
		}
		if l.Name == "LexForExpr" {
			return LexForExpr
		}
		// if reach eof throw while still in action throw error
		return l.Errorf("unclosed action")
	case lex.IsSpace(r):
		return Space
	case unicode.IsLetter(r): //variable and function must begin with a letter
		return Identifier
	case r == '!':
		if unicode.IsLetter(l.Peek()) {
			return Identifier
		}
		return l.Errorf("invalid character in builtin")
	case r == '#' || r == '+':
		if unicode.IsLetter(l.Peek()) {
			return EspraURI
		}
		return l.Errorf("invalid character in URI")
	case r == '-' || unicode.IsDigit(r):
		l.Backup()
		return Number
	case r == '\'' || r == '"':
		l.Emit(ItemOpenQuote)
		return String
	case r == '(':
		l.IntState[ActionParenDepth] += 1
		l.Emit(ItemLeftParen)
	case r == ')':
		l.IntState[ActionParenDepth] -= 1
		l.Emit(ItemRightParen)
	case r == '|':
		l.Emit(ItemPipe)
	default:
		return l.Errorf("Unexpected Character '%s'", string(r))
	}
	return InsideAction
}

func LexTextNode(l *lex.Lexer) lex.StateFn {
	for {
		r := l.Next()
		if lex.IsAlphaNumeric(l.Peek()) {
			if lex.IsSpace(r) || lex.IsEndOfLine(r) {
				l.Emit(ItemText)
			}
			i := lex.Pos(strings.IndexAny(l.Input[l.Pos:], " \t\n\r"))
			if i != -1 {
				i = l.Pos + i
			} else {
				i = lex.Pos(len(l.Input) - 1)
			}
			if strings.Contains(l.Input[l.Pos:i], ":") {
				l.Pos = i
				return LexURI
			}
		}
		l.Backup()
		if strings.HasPrefix(l.Input[l.Pos:], leftDelim) {
			if l.Pos > l.Start {
				l.Emit(ItemText)
			}
			return LeftDelim
		}
		if l.Next() == lex.EOF {
			break
		}
	}
	// Correctly reached EOF.
	if l.Pos > l.Start {
		l.Emit(ItemText)
	}
	l.Emit(lex.ItemEOF)
	return nil
}

// Quote scans a quoted string.
func String(l *lex.Lexer) lex.StateFn {
Loop:
	for {
		switch l.Next() {
		case '\\':
			// ???
			if r := l.Next(); r != lex.EOF && r != '\n' {
				break
			}
			fallthrough
		case lex.EOF, '\n':
			return l.Errorf("unterminated quoted string")
		case '"', '\'':
			l.Backup()
			break Loop
		}
	}
	l.Emit(ItemString)
	l.Next()
	l.Emit(ItemCloseQuote)
	return InsideAction
}