本文整理汇总了Python中tokenizer.Tokenizer.append方法的典型用法代码示例。如果您正苦于以下问题：Python Tokenizer.append方法的具体用法？Python Tokenizer.append怎么用？Python Tokenizer.append使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类tokenizer.Tokenizer的用法示例。
在下文中一共展示了Tokenizer.append方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: from tokenizer import Tokenizer [as 别名]
# 或者: from tokenizer.Tokenizer import append [as 别名]
class Interpreter:
def __init__(self, code_string=None):
self._KEYWORDS = ['read', 'write']
self._token = None
self._line = 0
self._tokenizer = Tokenizer(code_string, ['+','-','/','*','(',')',':='], ['\n',' '])
self._symboltable = SymbolTable()
def reset(self):
self._line = 0
self._token = None
self._tokenizer.clear()
def interpret(self, code_string=None):
if code_string is not None:
self._tokenizer.append(code_string)
self._consume()
self.program()
def _consume(self, _nomable=None):
if _nomable == '$$':
self.reset()
return True
if _nomable == 'id':
self._symboltable.add(self._token, self._line)
# TODO: add current token to AST
self._token = self._tokenizer.next()
def _is_token_id(self, _id=None):
if self._token is None:
raise ParseError(self._line, 'unexpected EOF')
if _id is None:
_id = self._token
if self._symboltable.has(_id):
return True
elif _id.isalpha() and _id not in self._KEYWORDS:
return True
else:
return False
def _is_token_num(self, _num=None):
if self._token is None:
raise ParseError(self._line, 'unexpected EOF')
if _num is None:
_num = self._token
if _num.isdigit():
return True
else:
return False
def _is_token_id_or_num(self, _token=None):
if _token is None:
_token = self._token
if self._is_token_id(_token) or self._is_token_num(_token):
return True
else:
return False
def _match(self, expected):
# TODO: might conflict with id's named 'id' or 'number'
if expected == self._token or expected in ['id', 'number']:
self._consume(self._token)
else:
raise TokenError(self._line, self._token, expected)
def _skip(self):
pass
def program(self):
if self._token in ['read', 'write', '$$'] or self._is_token_id():
self._stmt_list()
self._match('$$')
else:
raise ParseError(self._line, 'program')
def _stmt_list(self):
if self._token == '$$':
self._skip()
elif self._token in ['read', 'write'] or self._is_token_id():
self._line += 1
self._stmt()
self._stmt_list()
else:
raise ParseError(self._line, 'stmt_list')
def _stmt(self):
if self._token == 'read':
self._match('read')
self._match('id')
elif self._token == 'write':
self._match('write')
self._expr()
#.........这里部分代码省略.........