本文整理汇总了Python中Lexer.Lexer.lex方法的典型用法代码示例。如果您正苦于以下问题：Python Lexer.lex方法的具体用法？Python Lexer.lex怎么用？Python Lexer.lex使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Lexer.Lexer的用法示例。
在下文中一共展示了Lexer.lex方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestPyPOS
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import lex [as 别名]
class TestPyPOS(unittest.TestCase):
def setUp(self):
'''Tests Setup'''
self.lexer = Lexer()
self.tagger = POSTagger()
self.start = time.time()
def stringTest(self,string):
'''Common Testing Function'''
self.words = self.lexer.lex(string)
self.tags = self.tagger.tag(self.words)
self.end = time.time()
self.difference = self.end - self.start
for tag in self.tags:
print " / ".join(tag)
def test_1_Short(self):
'''Test Short String'''
global shortTestString
self.stringTest(shortTestString)
def test_2_Long(self):
'''Test Long String'''
global testString
self.stringTest(testString)
def tearDown(self):
print "Tokenized and tagged %s words in %s seconds" % (len(self.words),self.difference)
print "Running time at test end was: %s seconds" % (time.time() - STARTTIME)示例2: evaluate
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import lex [as 别名]
def evaluate(self):
#lexer
lexer = Lexer(self.input)
token_list = lexer.lex()
#parser
parser = Parser(token_list)
value = parser.parse()
return value示例3: __init__
# 需要导入模块: from Lexer import Lexer [as 别名]
# 或者: from Lexer.Lexer import lex [as 别名]
class Parser:
def __init__(self, filename):
self.lexer = Lexer(filename)
self.advance()
def check(self, name):
return (self.currLexeme.name == name)
def advance(self):
self.currLexeme = self.lexer.lex()
def matchNoAdvance(self, name):
if not self.check(name):
print("illegal")
print("expected ", name, ", but encountered ", self.currLexeme.name, self.currLexeme.value)
exit(0)
def match(self, name):
self.matchNoAdvance(name)
temp = self.currLexeme
if self.currLexeme.name != "ENDOFFILE":
self.advance()
return temp
def operator(self):
if self.check("PLUS"):
return self.match("PLUS")
elif self.check("TIMES"):
return self.match("TIMES")
elif self.check("DIVIDE"):
return self.match("DIVIDE")
elif self.check("SUBTRACT"):
return self.match("SUBTRACT")
elif self.check("MODULUS"):
return self.match("MODULUS")
elif self.check("AND"):
return self.match("AND")
elif self.check("OR"):
return self.match("OR")
elif self.check("XOR"):
return self.match("XOR")
elif self.check("EQUALS"):
return self.match("EQUALS")
elif self.check("NOTEQUALS"):
return self.match("NOTEQUALS")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("LESSTHAN"):
return self.match("LESSTHAN")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("GREATEROREQUAL"):
return self.match("GREATEROREQUAL")
elif self.check("LESSOREQUAL"):
return self.match("LESSOREQUAL")
def operatorPending(self):
return (self.check("PLUS") or
self.check("TIMES") or
self.check("DIVIDE") or
self.check("SUBTRACT") or
self.check("MODULUS") or
self.check("AND") or
self.check("OR") or
self.check("XOR") or
self.check("EQUALS") or
self.check("NOTEQUALS") or
self.check("GREATERTHAN") or
self.check("LESSTHAN") or
self.check("GREATEROREQUAL") or
self.check("LESSOREQUAL"))
def primary(self):
if self.check("NUMBER"):
return self.match("NUMBER")
elif self.check("STRING"):
return self.match("STRING")
elif self.check("OBRACKET"):
self.match("OBRACKET")
e = self.expression()
self.match("CBRACKET")
return e
elif self.arrayPending():
return self.array()
else:
return self.varExpression()
def primaryPending(self):
return (self.check("NUMBER") or
self.check("STRING") or
self.varExpressionPending() or
self.check("OBRACKET"))
def varExpression(self):
v = self.match("VARIABLE")
if self.check("OBRACKET"):
self.match("OBRACKET")
l = self.optList()
self.match("CBRACKET")
return cons("FUNCTION_CALL", v, l)
#.........这里部分代码省略.........