本文整理汇总了Python中lexer.Lexer.tok方法的典型用法代码示例。如果您正苦于以下问题：Python Lexer.tok方法的具体用法？Python Lexer.tok怎么用？Python Lexer.tok使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类lexer.Lexer的用法示例。
在下文中一共展示了Lexer.tok方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Parser
# 需要导入模块: from lexer import Lexer [as 别名]
# 或者: from lexer.Lexer import tok [as 别名]
class Parser(object):
def __init__(self, str, filename=None, **options):
self.input = str
self.lexer = Lexer(str, **options)
self.filename = filename
self.bloks = {}
self.options = options
self.contexts = [self]
self.extending = False
self._spaces = None
def context(self, parser):
if parser:
self.context.append(parser)
else:
self.contexts.pop()
def advance(self):
return self.lexer.advance()
def skip(self, n):
while n > 1: # > 0?
self.advance()
n -= 1
def peek(self):
p = self.lookahead(1)
return p
def line(self):
return self.lexer.lineno
def lookahead(self, n):
return self.lexer.lookahead(n)
def parse(self):
block = nodes.Block()
parser = None
block.line = self.line()
while "eos" != self.peek().type:
if "newline" == self.peek().type:
self.advance()
else:
block.append(self.parseExpr())
parser = self.extending
if parser:
self.context(parser)
ast = parser.parse()
self.context()
return ast
return block
def expect(self, type):
t = self.peek().type
if t == type:
return self.advance()
else:
raise Exception('expected "%s" but got "%s"' % (type, t))
def accept(self, type):
if self.peek().type == type:
return self.advance()
def parseExpr(self):
t = self.peek().type
if "yield" == t:
self.advance()
block = nodes.Block()
block._yield = True
return block
elif t in ("id", "class"):
tok = self.advance()
self.lexer.defer(self.lexer.tok("tag", "div"))
self.lexer.defer(tok)
return self.parseExpr()
funcName = "parse%s" % t.capitalize()
if hasattr(self, funcName):
return getattr(self, funcName)()
else:
raise Exception('unexpected token "%s"' % t)
def parseText(self):
tok = self.expect("text")
node = nodes.Text(tok.val)
node.line = self.line()
return node
def parseBlockExpansion(self):
if ":" == self.peek().type:
self.advance()
return nodes.Block(self.parseExpr())
else:
return self.block()
def parseAssignment(self):
#.........这里部分代码省略.........
示例2: Parser
# 需要导入模块: from lexer import Lexer [as 别名]
# 或者: from lexer.Lexer import tok [as 别名]
class Parser(object):
def __init__(self,str,filename=None,**options):
self.input = str
self.lexer = Lexer(str,**options)
self.filename = filename
self.bloks = {}
self.options = options
self.contexts = [self]
self.extending = False
self._spaces = None
def context(self,parser):
if parser: self.context.append(parser)
else: self.contexts.pop()
def advance(self):
return self.lexer.advance()
def skip(self,n):
while n>1: # > 0?
self.advance()
n -= 1
def peek(self):
p = self.lookahead(1)
return p
def line(self):
return self.lexer.lineno
def lookahead(self,n):
return self.lexer.lookahead(n)
def parse (self):
block = nodes.Block()
parser = None
block.line = self.line()
while 'eos' != self.peek().type:
if 'newline' == self.peek().type: self.advance()
else: block.append(self.parseExpr())
parser = self.extending
if parser:
self.context(parser)
ast = parser.parse()
self.context()
return ast
return block
def expect(self,type):
t = self.peek().type
if t == type: return self.advance()
else:
raise Exception('expected "%s" but got "%s" in file %s on line %d' %
(type, t, self.filename, self.line()))
def accept(self,type):
if self.peek().type == type: return self.advance()
def parseExpr(self):
t = self.peek().type
if 'yield' == t:
self.advance()
block = nodes.Block()
block._yield = True
return block
elif t in ('id','class'):
tok = self.advance()
self.lexer.defer(self.lexer.tok('tag','div'))
self.lexer.defer(tok)
return self.parseExpr()
funcName = 'parse%s'%t.capitalize()
if hasattr(self,funcName):
return getattr(self,funcName)()
else:
raise Exception('unexpected token "%s" in file %s on line %d' %
(t, self.filename, self.line()))
def parseText(self):
tok = self.expect('text')
node = nodes.Text(tok.val)
node.line = self.line()
return node
def parseBlockExpansion(self):
if ':'== self.peek().type:
self.advance()
return nodes.Block(self.parseExpr())
else:
return self.block()
def parseAssignment(self):
tok = self.expect('assignment')
return nodes.Assignment(tok.name,tok.val)
def parseCode(self):
#.........这里部分代码省略.........