本文整理汇总了Python中lexer.Lexer.parse_token方法的典型用法代码示例。如果您正苦于以下问题：Python Lexer.parse_token方法的具体用法？Python Lexer.parse_token怎么用？Python Lexer.parse_token使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类lexer.Lexer的用法示例。
在下文中一共展示了Lexer.parse_token方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Parser
# 需要导入模块: from lexer import Lexer [as 别名]
# 或者: from lexer.Lexer import parse_token [as 别名]
class Parser():
def __init__(self, data, name, errfn = None):
self._name = name
self._token = None
self._error = False
def report(msg):
print >> sys.stderr, msg
if not errfn: errfn = report
self._errfn = errfn
self._lexer = Lexer(data, name, errfn)
def parse(self):
return self.parse_grammar()
def _next_token(self):
while True:
self._token = self._lexer.parse_token()
if self._token.token == Token.WHITESPACE or \
self._token.token == Token.COMMENT: continue
break
return self._token
def _report_error(self, msg, token):
self._error = True
self._errfn("parser: %s[%s:%s]: %s" %\
(self._name, token.line, token.column, msg))
def parse_grammar(self):
self._next_token()
statements = []
while True:
tk = self._token
if tk.token == Token.END: break
stmt = self._parse_statement()
statements.append(stmt)
if not self._error:
return Grammar(statements)
return None
def _discard_until_semicolon(self):
while True:
tk = self._token
self._next_token()
if tk.token == Token.END or tk.token == Token.SEMICOLON:
break
def _parse_statement(self):
tk = self._token
if tk.token == Token.IDENTIFIER:
identifier = tk.tkstr
tk = self._next_token()
if tk.token == Token.COLON:
self._next_token()
expression = self._parse_expression()
tk = self._token
if tk.token == Token.SEMICOLON:
self._next_token()
return Statement(identifier, expression)
else:
msg = "expecting `;' at the end of statement"
else:
msg = "expecting `:' for statement"
else:
msg = "expecting identifier at the begining of statement"
self._report_error(msg, tk)
self._discard_until_semicolon()
return None
def _parse_expression(self):
components = []
comp = self._parse_component()
components.append(comp)
while self._token.token == Token.PIPE:
self._next_token()
comp = self._parse_component()
components.append(comp)
return Expression(components)
def _parse_component(self):
elements = []
while (not self._token.token == Token.PIPE) and\
(not self._token.token == Token.SEMICOLON) and\
(not self._token.token == Token.CLOSE_PARENTHESIS) and\
(not self._token.token == Token.END):
elm = self._parse_element()
elements.append(elm)
return Component(elements)
def _parse_element(self):
term = self._parse_term()
modifier = None
tk = self._token
if tk.token == Token.STAR or \
tk.token == Token.PLUS or \
tk.token == Token.QUESTION_MARK:
modifier = tk.tkstr
#.........这里部分代码省略.........