本文整理汇总了Python中flanker.addresslib.tokenizer.TokenStream.get_token方法的典型用法代码示例。如果您正苦于以下问题：Python TokenStream.get_token方法的具体用法？Python TokenStream.get_token怎么用？Python TokenStream.get_token使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类flanker.addresslib.tokenizer.TokenStream的用法示例。
在下文中一共展示了TokenStream.get_token方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _validate
# 需要导入模块: from flanker.addresslib.tokenizer import TokenStream [as 别名]
# 或者: from flanker.addresslib.tokenizer.TokenStream import get_token [as 别名]
def _validate(localpart):
stream = TokenStream(localpart)
while True:
# get alphanumeric portion
mpart = stream.get_token(ALPHANUM)
if mpart is None:
return False
# get optional dot, must be followed by more alphanumerics
mpart = stream.get_token(DOT)
if mpart is None:
break
# optional tags
tgs = _tags(stream)
if not stream.end_of_stream():
return False
return True示例2: _validate
# 需要导入模块: from flanker.addresslib.tokenizer import TokenStream [as 别名]
# 或者: from flanker.addresslib.tokenizer.TokenStream import get_token [as 别名]
def _validate(localpart):
stream = TokenStream(localpart)
# get the gmail base (alpha, num, or dot)
mpart = stream.get_token(GMAIL_BASE)
if mpart is None:
return False
# optional tags
tgs = _tags(stream)
if not stream.end_of_stream():
return False
return True示例3: _validate
# 需要导入模块: from flanker.addresslib.tokenizer import TokenStream [as 别名]
# 或者: from flanker.addresslib.tokenizer.TokenStream import get_token [as 别名]
def _validate(localpart):
stream = TokenStream(localpart)
# get the hotmail base
mpart = stream.get_token(HOTMAIL_BASE)
if mpart is None:
return False
# optional tags
tgs = _tags(stream)
if not stream.end_of_stream():
return False
return True示例4: _validate
# 需要导入模块: from flanker.addresslib.tokenizer import TokenStream [as 别名]
# 或者: from flanker.addresslib.tokenizer.TokenStream import get_token [as 别名]
def _validate(localpart):
"Grammar: local-part -> alpha { [ dot | underscore ] ( alpha | num ) }"
stream = TokenStream(localpart)
# local-part must being with alpha
alpa = stream.get_token(ALPHA)
if alpa is None:
return False
while True:
# optional dot or underscore token
stream.get_token(DOT) or stream.get_token(UNDERSCORE)
# alpha or numeric
alpanum = stream.get_token(ALPHA) or stream.get_token(NUMERIC)
if alpanum is None:
break
# alpha or numeric must be end of stream
if not stream.end_of_stream():
return False
return True示例5: _AddressParser
# 需要导入模块: from flanker.addresslib.tokenizer import TokenStream [as 别名]
# 或者: from flanker.addresslib.tokenizer.TokenStream import get_token [as 别名]
#.........这里部分代码省略.........
addrs = flanker.addresslib.address.AddressList()
unparsable = []
# address
addr = self._address()
if addr is None:
# synchronize to the next delimiter (or end of line)
# append the skipped over text to the unparsable list
skip = self.stream.synchronize()
if skip:
unparsable.append(skip)
# if no mailbox and end of stream, we were unable
# return the unparsable stream
if self.stream.end_of_stream():
return [], unparsable
else:
# if we found a delimiter or end of stream, we have a
# valid mailbox, add it
if self.stream.peek(DELIMITER) or self.stream.end_of_stream():
addrs.append(addr)
else:
# otherwise snychornize and add it the unparsable array
skip = self.stream.synchronize()
if skip:
pre = self.stream.stream[:self.stream.stream.index(skip)]
unparsable.append(pre + skip)
# if we hit the end of the stream, return the results
if self.stream.end_of_stream():
return [], [self.stream.stream]
while True:
# delimiter
dlm = self.stream.get_token(DELIMITER)
if dlm is None:
skip = self.stream.synchronize()
if skip:
unparsable.append(skip)
if self.stream.end_of_stream():
break
# address
start_pos = self.stream.position
addr = self._address()
if addr is None:
skip = self.stream.synchronize()
if skip:
unparsable.append(skip)
if self.stream.end_of_stream():
break
else:
# if we found a delimiter or end of stream, we have a
# valid mailbox, add it
if self.stream.peek(DELIMITER) or self.stream.end_of_stream():
addrs.append(addr)
else:
# otherwise snychornize and add it the unparsable array
skip = self.stream.synchronize()
if skip:
sskip = self.stream.stream[start_pos:self.stream.position]
unparsable.append(sskip)
# if we hit the end of the stream, return the results
if self.stream.end_of_stream():
return addrs, unparsable