本文整理匯總了Python中nltk.corpus.reader.SeekableUnicodeStreamReader方法的典型用法代碼示例。如果您正苦於以下問題：Python reader.SeekableUnicodeStreamReader方法的具體用法？Python reader.SeekableUnicodeStreamReader怎麽用？Python reader.SeekableUnicodeStreamReader使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類nltk.corpus.reader的用法示例。
在下文中一共展示了reader.SeekableUnicodeStreamReader方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: check_reader
# 需要導入模塊: from nltk.corpus import reader [as 別名]
# 或者: from nltk.corpus.reader import SeekableUnicodeStreamReader [as 別名]
def check_reader(unicode_string, encoding, n=1000):
bytestr = unicode_string.encode(encoding)
strlen = len(unicode_string)
stream = BytesIO(bytestr)
reader = SeekableUnicodeStreamReader(stream, encoding)
# Find all character positions
chars = []
while True:
pos = reader.tell()
chars.append( (pos, reader.read(1)) )
if chars[-1][1] == '': break
# Find all strings
strings = dict( (pos,'') for (pos,c) in chars )
for pos1, char in chars:
for pos2, _ in chars:
if pos2 <= pos1:
strings[pos2] += char
while True:
op = random.choice('tsrr')
# Check our position?
if op == 't': # tell
reader.tell()
# Perform a seek?
if op == 's': # seek
new_pos = random.choice([p for (p,c) in chars])
reader.seek(new_pos)
# Perform a read?
if op == 'r': # read
if random.random() < .3: pos = reader.tell()
else: pos = None
if random.random() < .2: size = None
elif random.random() < .8:
size = random.randint(0, int(strlen/6))
else: size = random.randint(0, strlen+20)
if random.random() < .8:
s = reader.read(size)
else:
s = reader.readline(size)
# check that everything's consistent
if pos is not None:
assert pos in strings
assert strings[pos].startswith(s)
n -= 1
if n == 0:
return 'passed'
#Call the randomized test function `check_reader` with a variety of
#input strings and encodings.