Golang lexer.Lexer類代碼示例

func makeSynthRules(l lexer.Lexer, synthSymbols map[string]bool, userFn UserFn) ([]Rule, error) {
	var rules []Rule
	if userFn == nil {
		userFn = DefaultUserFn
	}
	for tok := range synthSymbols {
		if !isSynthName(tok) {
			tlog.Panic("should be a synth name but it's not: ", tok)
		}
		lsid, err := l.GetSymbolSet().GetByName(tok)
		if err != nil {
			return rules, err
		}
		end := tok[len(tok)-1:]
		tok := tok[:len(tok)-1]
		rsid, err := rhsCase(l, tok)
		if err != nil {
			return rules, err
		}
		switch end {
		case "?":
			rules = append(rules, Rule{lsid, []syms.SymbolID{syms.EMPTY}, userFn})
			rules = append(rules, Rule{lsid, []syms.SymbolID{rsid}, userFn})
		case "*":
			rules = append(rules, Rule{lsid, []syms.SymbolID{syms.EMPTY}, userFn})
			rules = append(rules, Rule{lsid, []syms.SymbolID{lsid, rsid}, userFn})
		case "+":
			rules = append(rules, Rule{lsid, []syms.SymbolID{rsid}, userFn})
			rules = append(rules, Rule{lsid, []syms.SymbolID{lsid, rsid}, userFn})
		}
	}

	return rules, nil
}

func (p *Parser) nextToken(lex lexer.Lexer) (lexer.Token, error) {
	tok, err := lex.NextToken()
	if err == io.EOF {
		err = nil
		tok.ID = syms.EOF
	}
	return tok, err
}

func lhsCase(l lexer.Lexer, lhs string) (syms.SymbolID, error) {
	symbols := l.GetSymbolSet()
	sid, err := symbols.GetByName(lhs)
	if err == nil {
		return sid, nil
	}
	return symbols.Add(lhs, false)
}

func rhsCase(l lexer.Lexer, tok string) (syms.SymbolID, error) {
	if tok == "" {
		return syms.ERROR, errors.New("empty rhs identifier")
	}
	symbols := l.GetSymbolSet()
	sid, err := symbols.GetByName(tok)
	if err == nil {
		return sid, nil
	}

	lt := len(tok)
	if isIdentName(tok) {
		return l.Ident(tok[1 : lt-1]), nil
	} else if isOperatorName(tok) {
		return l.Operator(tok[1 : lt-1]), nil
	}
	if tok == "" || !tokenRe.MatchString(tok) {
		return syms.ERROR, fmt.Errorf("%s is not a valid rhs identifier", tok)
	}
	sid, err = symbols.Add(tok, false)
	return sid, err
}

func (p *Parser) Parse(lex lexer.Lexer) (interface{}, error) {
	defer tlog.FuncLog(tlog.Func("Parse"))

	stateStack := []ItemSetID{ItemSetID(0)}
	dataStack := make([]interface{}, 1, 1) // first is nil
	tok, err := p.nextToken(lex)
	if err != nil {
		return tok, err
	}
	if p.debug {
		fmt.Println("\nInitial token", tok.String(p.Syms), p.fmtStacks(stateStack, dataStack))
	}
	for {
		topState := stateStack[len(stateStack)-1]
		action, found := p.parsetable[topState][tok.ID]
		if !found {
			action, found = p.parsetable[topState][syms.EMPTY]
			if found {
				lex.Unread()
				tok = lexer.Token{ID: syms.EMPTY, Str: ""}
			}
		}
		if !found {
			// parsing error
			tok, stateStack, dataStack, err = p.errorRecovery(lex, tok, stateStack, dataStack)
			_ = "breakpoint"
			if err != nil {
				return dataStack, err
			}
			topState = stateStack[len(stateStack)-1]
			action, found = p.parsetable[topState][tok.ID]
		}

		if action.actiontype == Shift {
			stateStack = append(stateStack, action.state)
			dataStack = append(dataStack, tok)
			if p.debug {
				fmt.Println("\nShifted token", tok.String(p.Syms), p.fmtStacks(stateStack, dataStack))
			}
			tok, err = p.nextToken(lex)
			if err != nil {
				dataStack = append(dataStack, tok)
				return dataStack, err
			}
		} else if action.actiontype == Reduce {
			rule := p.Rules.Get(action.ruleID)

			cutoff := len(stateStack) - len(rule.RHS)
			if cutoff < 0 {
				tlog.Panic("should reduce, but not enough elements on stack.",
					"\nReduce before token ", tok.String(p.Syms),
					"\nRule: ", rule.String(*p.Syms, -1),
					p.fmtStacks(stateStack, dataStack))
			}

			userFn := rule.UserFn
			if userFn == nil {
				userFn = grammar.NilUserFn
			}
			newdata, err := userFn(p.Grammar, rule, dataStack[cutoff:])
			if err != nil {
				return newdata, UserCallbackParseError(
					fmt.Sprint("user callback error on reduce. ",
						"\nReduce before token ", tok.String(p.Syms),
						"\nRule: ", rule.String(*p.Syms, -1),
						p.fmtStacks(stateStack, dataStack)))
			}
			dataStack = dataStack[:cutoff]
			dataStack = append(dataStack, newdata)

			stateStack = stateStack[:cutoff]
			topState = stateStack[len(stateStack)-1]
			if topState == 0 && len(stateStack) == 1 && tok.ID == syms.EOF {
				if p.debug {
					fmt.Println(
						"\nFinal Reduce, before token ", tok.String(p.Syms),
						"\nRule: ", rule.String(*p.Syms, -1),
						"\nReduced to state:", topState,
						p.fmtStacks(stateStack, dataStack))
				}
				return newdata, nil
			}

			action2, found := p.parsetable[topState][rule.LHS]
			if !found {
				tlog.Panic("internal parser error, no goto state. ",
					"\nAfter Reduce, before token ", tok.String(p.Syms),
					"\nRule: ", rule.String(*p.Syms, -1),
					p.fmtStacks(stateStack, dataStack))
			}
			if action2.actiontype != Goto {
				tlog.Panic("internal parser error, expected goto action, instead got: ", action2,
					"\nAfter Reduce, before token ", tok.String(p.Syms),
					"\nRule: ", rule.String(*p.Syms, -1),
					p.fmtStacks(stateStack, dataStack))
			}
			stateStack = append(stateStack, action2.state)
			// a reduce does not consume the terminal

			if p.debug {
//.........這裏部分代碼省略.........

func (p *Parser) errorRecovery(lex lexer.Lexer, tok lexer.Token, stateStack []ItemSetID, dataStack []interface{}) (lexer.Token, []ItemSetID, []interface{}, error) {
	parseErr := ParseError{
		Invalid:  []lexer.Token{tok},
		Location: lex.Location(),
	}
	topState := stateStack[len(stateStack)-1]
	for expectedTok := range p.parsetable[topState] {
		parseErr.Expected = append(parseErr.Expected, p.Syms.Get(expectedTok))
	}

	// rewind the state stack searching for an error rule
	action, found := p.parsetable[topState][syms.ERROR]
	for !(found && action.actiontype == Shift) {
		stateStack = stateStack[:len(stateStack)-1]
		if len(stateStack) == 0 {
			break
		}
		topState = stateStack[len(stateStack)-1]
		action, found = p.parsetable[topState][syms.ERROR]
	}

	parseErr.Valid = append(parseErr.Valid, dataStack[len(stateStack):]...)
	dataStack = dataStack[:len(stateStack)]
	if len(stateStack) == 0 {
		err := UnexpectedTerminalParseError("parse error; could not find a suitable error rule")
		dataStack = append(dataStack, parseErr)
		if p.debug {
			fmt.Println("\nParse error, no error recovery possible", p.fmtStacks(stateStack, dataStack))
		}
		return tok, stateStack, dataStack, err
	}
	stateStack = append(stateStack, action.state)

	// now search for next action by discarding unfitting tokens
	actionMap := p.parsetable[action.state]
	action, found = actionMap[tok.ID]
	parseErr.Invalid = nil
	for !found {
		parseErr.Invalid = append(parseErr.Invalid, tok)
		if tok.ID == syms.EOF {
			err := UnexpectedTerminalParseError("parse error; could not find a suitable error rule before EOF")
			dataStack = append(dataStack, parseErr)
			if p.debug {
				fmt.Println("\nParse error, EOF while recovering",
					"\nRecovery state expected one of:", p.symNames(actionMap),
					p.fmtStacks(stateStack, dataStack))
			}
			return tok, stateStack, dataStack, err
		}
		var err error
		tok, err = p.nextToken(lex)
		if err != nil {
			dataStack = append(dataStack, parseErr)
			return tok, stateStack, dataStack, err
		}
		action, found = actionMap[tok.ID]
	}
	dataStack = append(dataStack, parseErr)
	if p.debug {
		fmt.Println("\nParse error, stack unwinded", p.fmtStacks(stateStack, dataStack))
	}
	return tok, stateStack, dataStack, nil
}

func NewGrammarBlock(l lexer.Lexer, ruleBlock string, funcMap map[string]UserFn) (Grammar, error) {
	// defer tlog.FuncLog(tlog.Func("NewGrammarBlock"))
	var rules []Rule
	arrowSep := "→"
	defineSep := "::="
	subruleSep := "|"
	ruleHandlerPrefix := "@"
	ruleList := strings.Split(ruleBlock, "\n")
	synthSymbols := make(map[string]bool)
	var lhsID syms.SymbolID
	var err error
	for lineno, r := range ruleList {
		r = strings.TrimSpace(r)
		if r == "" || strings.HasPrefix(r, "//") {
			continue
		}

		var rhs string
		if lhsID != syms.ERROR && strings.HasPrefix(r, subruleSep) {
			// new subrule, lhs stays the same
			rhs = r[len(subruleSep):]
		} else {
			// new rule
			idxArrow := strings.Index(r, arrowSep)
			idxDefine := strings.Index(r, defineSep)

			var lhs string
			if idxArrow < 0 && idxDefine < 0 {
				return Grammar{}, fmt.Errorf("rule on line %d has no '→' or '::=' symbol", lineno+1)
			} else if idxArrow >= 0 && (idxDefine < 0 || idxArrow < idxDefine) {
				lhs = r[:idxArrow]
				rhs = r[idxArrow+len(arrowSep):]
			} else {
				lhs = r[:idxDefine]
				rhs = r[idxDefine+len(defineSep):]
			}

			lhsTrim := strings.TrimSpace(lhs)
			if lhsTrim == "" || !identRe.MatchString(lhsTrim) {
				return Grammar{}, fmt.Errorf("line %d: '%s' is not a valid lhs identifier\n%s", lineno+1, lhs, r)
			}
			lhsID, err = lhsCase(l, lhsTrim)
			if err != nil {
				return Grammar{}, err
			}
		}

		var rhsa []syms.SymbolID
		var funcDecl string
		for _, tok := range strings.Fields(rhs) {
			if tok == subruleSep {
				if len(rhsa) == 0 {
					return Grammar{}, fmt.Errorf("line %d: empty partial rule\n%s", lineno+1, r)
				}
				rules = append(rules, Rule{lhsID, rhsa, funcMap[funcDecl]})
				rhsa = nil
				funcDecl = ""
			} else if strings.HasPrefix(tok, ruleHandlerPrefix) {
				if funcDecl != "" {
					return Grammar{}, fmt.Errorf("line %d: multiple function decls\n%s", lineno+1, r)
				}
				funcDecl = tok[len(ruleHandlerPrefix):]
			} else {
				sid, err := rhsCase(l, tok)
				if err != nil {
					return Grammar{}, fmt.Errorf("line %d: %s\n%s", lineno+1, err, r)
				}
				rhsa = append(rhsa, sid)
				if isSynthName(tok) {
					synthSymbols[tok] = true
				}
			}
		}
		if len(rhsa) == 0 {
			return Grammar{}, fmt.Errorf("line %d: empty rhs\n%s", lineno+1, r)
		}
		rules = append(rules, Rule{lhsID, rhsa, funcMap[funcDecl]})
	}
	newRules, err := makeSynthRules(l, synthSymbols, funcMap["synth"])
	if err != nil {
		return Grammar{}, nil
	}
	rules = append(rules, newRules...)

	return NewGrammar(l.GetSymbolSet(), rules)
}