本文整理汇总了Python中nltk.corpus.reader.SeekableUnicodeStreamReader方法的典型用法代码示例。如果您正苦于以下问题：Python reader.SeekableUnicodeStreamReader方法的具体用法？Python reader.SeekableUnicodeStreamReader怎么用？Python reader.SeekableUnicodeStreamReader使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类nltk.corpus.reader的用法示例。
在下文中一共展示了reader.SeekableUnicodeStreamReader方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: check_reader
# 需要导入模块: from nltk.corpus import reader [as 别名]
# 或者: from nltk.corpus.reader import SeekableUnicodeStreamReader [as 别名]
def check_reader(unicode_string, encoding, n=1000):
bytestr = unicode_string.encode(encoding)
strlen = len(unicode_string)
stream = BytesIO(bytestr)
reader = SeekableUnicodeStreamReader(stream, encoding)
# Find all character positions
chars = []
while True:
pos = reader.tell()
chars.append( (pos, reader.read(1)) )
if chars[-1][1] == '': break
# Find all strings
strings = dict( (pos,'') for (pos,c) in chars )
for pos1, char in chars:
for pos2, _ in chars:
if pos2 <= pos1:
strings[pos2] += char
while True:
op = random.choice('tsrr')
# Check our position?
if op == 't': # tell
reader.tell()
# Perform a seek?
if op == 's': # seek
new_pos = random.choice([p for (p,c) in chars])
reader.seek(new_pos)
# Perform a read?
if op == 'r': # read
if random.random() < .3: pos = reader.tell()
else: pos = None
if random.random() < .2: size = None
elif random.random() < .8:
size = random.randint(0, int(strlen/6))
else: size = random.randint(0, strlen+20)
if random.random() < .8:
s = reader.read(size)
else:
s = reader.readline(size)
# check that everything's consistent
if pos is not None:
assert pos in strings
assert strings[pos].startswith(s)
n -= 1
if n == 0:
return 'passed'
#Call the randomized test function `check_reader` with a variety of
#input strings and encodings.