本文整理汇总了Python中scanner.Scanner.nextToken方法的典型用法代码示例。如果您正苦于以下问题：Python Scanner.nextToken方法的具体用法？Python Scanner.nextToken怎么用？Python Scanner.nextToken使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scanner.Scanner的用法示例。
在下文中一共展示了Scanner.nextToken方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: main
# 需要导入模块: from scanner import Scanner [as 别名]
# 或者: from scanner.Scanner import nextToken [as 别名]
def main(*args):
scanner = Scanner()
symbolTable = SymbolTable()
while True:
# Get the next token from scanner
token = scanner.nextToken()
# Pretty print the token
token.__repr__()
if token.TokenCode == 'tc_ID':
# Check if token exists in SymbolTable
entry = symbolTable.lookup(token.DataValue[0].lower())
if entry == -1 : # -1 means not found in table
# Entry does not exist -> add it!
num = symbolTable.insert(token.DataValue[0].lower())
# Associate the token with the entry
token.setSymTabEntry(num)
else:
# Token exists:
# Associate the token with the entry
token.setSymTabEntry(entry)
elif token.TokenCode == 'tc_NUMBER':
# Same as for entry ..
entry = symbolTable.lookup(token.DataValue[0].lower())
if entry == -1:
num = symbolTable.insert(token.DataValue[0].lower())
token.setSymTabEntry(num)
else:
token.setSymTabEntry(entry)
elif token.TokenCode == 'tc_EOF':
# Reached end of input -> quit loop!
break
# Pretty print our table
symbolTable.__repr__()示例2: __init__
# 需要导入模块: from scanner import Scanner [as 别名]
# 或者: from scanner.Scanner import nextToken [as 别名]
class Parser:
global depth
def __init__(self,input):
self.__scanner = Scanner(input)
self.__currentToken = None
self.__foundError = False
self.__errorInFunction = ''
self.__errorDepth = None
self.printTree = False
self.errors = []
self.skipped = []
self.symbolTable = symbolTable.SymbolTable()
def parse(self,bool=False):
if bool: self.printTree = True
self.__getToken()
self.__Program()
def __getToken(self):
self.__currentToken = self.__scanner.nextToken()
if not self.__currentToken: return False
if self.__currentToken.TokenCode == 'tc_ID':
# Check if token exists in SymbolTable
entry = self.symbolTable.lookup(self.__currentToken.DataValue[0].upper())
if entry == -1 : # -1 means not found in table
# Entry does not exist -> add it!
num = self.symbolTable.insert(self.__currentToken.DataValue[0].upper(),self.__currentToken.TokenCode)
# Associate the token with the entry
self.__currentToken.setSymTabEntry(num)
else:
# Token exists:
# Associate the token with the entry
self.__currentToken.setSymTabEntry(entry)
elif self.__currentToken.TokenCode == 'tc_NUMBER':
# Same as for entry ..
entry = self.symbolTable.lookup(self.__currentToken.DataValue[0].upper())
if entry == -1:
num = self.symbolTable.insert(self.__currentToken.DataValue[0].upper(),self.__currentToken.TokenCode)
self.__currentToken.setSymTabEntry(num)
else:
self.__currentToken.setSymTabEntry(entry)
return True
def __callersname(self):
return sys._getframe(2).f_code.co_name
def __addError(self,token,message):
self.errors.append(Error(token.lineno,token.columnno,message))
def __recover(self):
global depth
if self.printTree: print '\t',' '*depth,'->Trying to recover.'
if len(syncsets[self.__errorInFunction]) == 0: return
while True:
if not self.__currentToken.TokenCode in syncsets[self.__errorInFunction]:
if self.printTree: print '\t',' '*depth, '-->Discarded %s'%self.__currentToken.DataValue[0]
self.__getToken()
if not self.__currentToken:
if self.printTree: print '\t',' '*(depth),'->Reached end of input, could not recover.'
break
else:
if self.printTree: print '\t',' '*depth, '-->Match: %s with %s'%(self.__currentToken.TokenCode,self.__currentToken.TokenCode)
self.__foundError = False
self.__errorInFunction = ''
self.__errorDepth = None
if self.printTree: print '\t',' '*depth,'->Recovered!'
#self.__getToken()
break
def __match(self,expectedIn):
# To track error depth - we do not want to recover from sibling or
# parent's functions.
global depth
# If we encounter an illegal symbol, skip passed it and report.
if self.__currentToken.TokenCode == 'tc_ERROR':
message = 'Illegal character'
while self.__currentToken.TokenCode == 'tc_ERROR':
self.__addError(self.__currentToken,message)
self.__getToken()
# If we know there is an error, check depth and exit if depth is the same
# or greater. Recover if error depth is smaller.
if self.__foundError:
if self.__errorDepth < depth:
if self.__currentToken.TokenCode in syncsets[self.__errorInFunction]:
if self.printTree: print '\t',' '*depth,'--->Skipping', self.__currentToken.TokenCode, 'because of the Error flag.'
return
else:
#if self.printTree: print '\t',' '*depth,self.__errorInFunction, syncsets[self.__errorInFunction]
self.__recover()
#.........这里部分代码省略.........
示例3: __init__
# 需要导入模块: from scanner import Scanner [as 别名]
# 或者: from scanner.Scanner import nextToken [as 别名]
class compParser:
global depth
def __init__(self,input):
self.__scanner = Scanner(input)
self.__code = code.Code()
self.__currentToken = None
self.__foundError = False
self.__errorInFunction = ''
self.__errorDepth = None
self.__tempList = []
self.printTree = False
self.errors = []
self.SymbolTable = SymbolTable()
self.SymbolTable.insert('0','tc_CONST')
self.SymbolTable.insert('1','tc_CONST')
def __newTemp(self):
# Create a new temp variable, insert into SymbolTable and return temp name
varName = self.__code.newTemp()
stPointer = self.SymbolTable.insert(varName,'tc_VAR')
self.__code.generate('cd_VAR',None,None,self.SymbolTable.SymbolTable[stPointer].m_lexeme)
return stPointer
def __newLabel(self):
# Create a new label name and return label name
labelName = self.__code.newLabel()
stPointer = self.SymbolTable.insert(labelName,'tc_LABEL')
self.__code.generate('cd_LABEL',None,None,self.SymbolTable.SymbolTable[stPointer].m_lexeme)
def printTacToFile(self,fileName):
self.__code.printTacToFile(fileName)
def printCode(self):
self.__code.__repr__()
def parse(self,bool=False):
if bool: self.printTree = True
self.__getToken()
self.__Program()
def __getTokenCode(self):
return self.__currentToken.TokenCode
def __getToken(self):
self.__currentToken = self.__scanner.nextToken()
if self.__currentToken == 'tc_EOF': return False
if self.__getTokenCode() == 'tc_ID':
entry = self.SymbolTable.lookup(self.__currentToken.DataValue[0].upper())
if entry == -1 : # -1 means not found in table
# Entry does not exist -> add it!
num = self.SymbolTable.insert(self.__currentToken.DataValue[0].upper(),self.__getTokenCode())
# Associate the token with the entry
self.__currentToken.setSymTabEntry(num)
else:
# Token exists:
# Associate the token with the entry
self.__currentToken.setSymTabEntry(entry)
elif self.__getTokenCode() == 'tc_NUMBER':
# Same as for entry ..
entry = self.SymbolTable.lookup(self.__currentToken.DataValue[0].upper())
if entry == -1:
num = self.SymbolTable.insert(self.__currentToken.DataValue[0].upper(),self.__getTokenCode())
self.__currentToken.setSymTabEntry(num)
else:
self.__currentToken.setSymTabEntry(entry)
return True
def __callersname(self):
# Returns the name of the calling function
return sys._getframe(2).f_code.co_name
def __addError(self,token,message):
self.errors.append(Error(token.lineno,token.columnno,message))
def __recover(self):
global depth
if self.printTree: print '\t',' '*depth,'->Trying to recover.'
if len(syncsets[self.__errorInFunction]) == 0: return
while True:
if not self.__getTokenCode() in syncsets[self.__errorInFunction]:
if self.printTree: print '\t',' '*depth, '-->Discarded %s'%self.__currentToken.DataValue[0]
result = self.__getToken()
if not result:
if self.printTree: print '\t',' '*(depth),'->Reached end of input, could not recover.'
break
else:
if self.printTree: print '\t',' '*depth, '-->Match: %s with %s'%(self.__getTokenCode(),self.__getTokenCode())
self.__foundError = False
self.__errorInFunction = ''
self.__errorDepth = None
if self.printTree: print '\t',' '*depth,'->Recovered!'
break
def __match(self,expectedIn):
# To track error depth - we do not want to recover from sibling or
# parent's functions.
global depth
#.........这里部分代码省略.........