本文整理汇总了Python中Lexer.Lexer.tokenize方法的典型用法代码示例。如果您正苦于以下问题：Python Lexer.tokenize方法的具体用法？Python Lexer.tokenize怎么用？Python Lexer.tokenize使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Lexer.Lexer的用法示例。
在下文中一共展示了Lexer.tokenize方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TokenStream
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import tokenize [as 别名]
class TokenStream(unittest.TestCase):
def setUp(self):
self.lexer = Lexer()
def assert_token_stream_has_not_ended(self, token_stream):
assert bool(token_stream)
assert not token_stream.has_ended()
def assert_token_type_correspond(self, token, token_type):
self.assertTrue(token.test(token_type))
def test_can_iterate_over_token_stream(self):
source = '{{ name }}'
token_stream = self.lexer.tokenize(source)
self.assert_token_type_correspond(token_stream.current, TOKEN_VARIABLE_START)
self.assert_token_stream_has_not_ended(token_stream)
next(token_stream)
self.assert_token_type_correspond(token_stream.current, TOKEN_NAME + ':' + 'name')
self.assert_token_stream_has_not_ended(token_stream)
next(token_stream)
self.assert_token_type_correspond(token_stream.current, TOKEN_VARIABLE_END)
self.assert_token_stream_has_not_ended(token_stream)
next(token_stream)
self.assert_token_type_correspond(token_stream.current, TOKEN_EOF)
assert not bool(token_stream)
assert token_stream.has_ended()
def test_token_stream_has_right_types(self):
token_types = [TOKEN_VARIABLE_START, TOKEN_NAME, TOKEN_VARIABLE_END,
TOKEN_DATA, TOKEN_BLOCK_START, TOKEN_NAME, TOKEN_BLOCK_END]
source = '{{ data }}hello{%foo%}'
self.assertListEqual(token_types, [token.token_type for token in self.lexer.tokenize(source)])
def test_stops_iteration_when_stream_ends(self):
source = 'hello {{name}} {%foo%}{{bar}}baz{%end%}'
stream = self.lexer.tokenize(source)
while stream:
next(stream)
assert stream.has_ended()
assert not bool(stream)
assert not stream示例2: get_expression
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import tokenize [as 别名]
def get_expression(self, expr, check=0):
def interpret(expression, pr, action):
lval, rval = self.get_operand(self.tokens[self.dp][pr-1]), self.get_operand(self.tokens[self.dp][pr+1])
if (Token.T_CINT in lval) and (Token.T_CINT in rval):
replace = {
"^" : "**",
"mod" : "%",
"and" : "&",
"or" : "|",
"xor" : "^"
}
try: self.tokens[self.dp][pr] = replace[self.tokens[self.dp][pr]]
except: pass
self.tokens[self.dp][pr] = str(eval(lval.value+self.tokens[self.dp][pr]+rval.value))
self.types.append(Token.T_CINT)
else:
# DEBUG ONLY
# without const flag
if not lval.type: lval.type = rval.type & ~Token.T_CONST
elif not rval.type: rval.type = lval.type & ~Token.T_CONST
for t in Parser.valid_types:
if (lval.type & t[0] == t[0]) and (rval.type & t[1] == t[1]) and (self.tokens[self.dp][pr] in t[3]):
self.types.append(t[2])
flag = True
break
try: flag
except: raise TypeError, "incorrect types in expression '%s'" % expr
# DEBUG ONLY
if not lval.value: lval.type = 0
if not rval.value: rval.type = 0
action(expression, lval, rval)
self.tokens[self.dp][pr] = None
del self.tokens[self.dp][pr-1], self.tokens[self.dp][pr]
# change indexes in priority
self.priority[self.dp] = [x-2 if x>pr else x for x in self.priority[self.dp]]
expression = Token(Token.T_EXPRESSION)
self.types = []
self.dp += 1
try: self.tokens[self.dp] = []
except: self.tokens.append([])
try: self.priority[self.dp] = []
except: self.priority.append([])
self.tokens[self.dp], self.priority[self.dp] = Lexer.tokenize(expr, Parser.constants)
if len(self.tokens[self.dp]) == 1:
op = self.get_operand(self.tokens[self.dp][0], check)
if Token.T_FUNCTION in op:
return Token(Token.T_EXPRESSION | op.type) << op
else:
self.dp -= 1
return op
for x in xrange(len(self.priority[self.dp])):
pr = self.priority[self.dp][x]
op = self.tokens[self.dp][pr]
interpret(expression, pr, lambda e,x,y: e << Token(Token.T_OPERATOR, op) << x << y)
expression.type |= self.types[0]
try:
int(self.tokens[self.dp][0])
# it is const int expression
return expression << Token() << Token() << Token(Token.T_CINT, self.tokens[self.dp][0])
except: return expression
self.dp -= 1
return expression示例3: Environment
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import tokenize [as 别名]
class Environment(object):
def __init__(self):
self.lexer = Lexer()
def tokenize(self, source):
return self.lexer.tokenize(source)示例4: LexerTest
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import tokenize [as 别名]
class LexerTest(unittest.TestCase):
def setUp(self):
self.lexer = Lexer()
def assert_tokens_are_correct(self, tokens, source):
self.assertListEqual(tokens, self.get_tokens(source))
def get_tokens(self, source):
return [token.items() for token in self.lexer.tokenize(source)]
def assert_stream_raises(self, stream, exception):
try:
for _ in stream:
pass
self.assertTrue(False)
except exception:
pass
def test_empty_text_return_no_tokens(self):
self.assert_tokens_are_correct([], '')
def test_can_parse_raw_text(self):
source = 'raw_text'
tokens = [(TOKEN_DATA, source)]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_start_block(self):
source = '{%'
tokens = [(TOKEN_BLOCK_START, source)]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_end_block(self):
source = '{{'
tokens = [(TOKEN_VARIABLE_START, source)]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_entire_block(self):
source = '{% bla bla %}'
tokens = [
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'bla'),
(TOKEN_NAME, 'bla'),
(TOKEN_BLOCK_END, '%}')
]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_entire_variable(self):
source = '{{ bla }}'
tokens = [
(TOKEN_VARIABLE_START, '{{'),
(TOKEN_NAME, 'bla'),
(TOKEN_VARIABLE_END, '}}')
]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_start_block_and_end_block(self):
source = '{% bla %}hello{% end %}'
tokens = [
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'bla'),
(TOKEN_BLOCK_END, '%}'),
(TOKEN_DATA, 'hello'),
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'end'),
(TOKEN_BLOCK_END, '%}')
]
self.assert_tokens_are_correct(tokens, source)
def test_can_parse_nested_block(self):
source = '{%foo%}{%bar%} {{hello}} {%end%}{%end%}'
tokens = [
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'foo'),
(TOKEN_BLOCK_END, '%}'),
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'bar'),
(TOKEN_BLOCK_END, '%}'),
(TOKEN_DATA, ' '),
(TOKEN_VARIABLE_START, '{{'),
(TOKEN_NAME, 'hello'),
(TOKEN_VARIABLE_END, '}}'),
(TOKEN_DATA, ' '),
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'end'),
(TOKEN_BLOCK_END, '%}'),
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'end'),
(TOKEN_BLOCK_END, '%}')
]
self.assert_tokens_are_correct(tokens, source)
def can_parse_consecutive_blocks(self):
source = '{%foo%}{%end%}{{baz}}{%bar%}{%end%}'
tokens = [
(TOKEN_BLOCK_START, '{%'),
(TOKEN_NAME, 'foo'),
(TOKEN_BLOCK_END, '%}'),
(TOKEN_BLOCK_START, '{%'),
#.........这里部分代码省略.........