本文整理汇总了Python中dictionary.Dictionary.filter_tokens方法的典型用法代码示例。如果您正苦于以下问题：Python Dictionary.filter_tokens方法的具体用法？Python Dictionary.filter_tokens怎么用？Python Dictionary.filter_tokens使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dictionary.Dictionary的用法示例。
在下文中一共展示了Dictionary.filter_tokens方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Corpus
# 需要导入模块: from dictionary import Dictionary [as 别名]
# 或者: from dictionary.Dictionary import filter_tokens [as 别名]
class Corpus(object):
"""
"""
def __init__(self, document_generator, stop_words):
self.document_generator = document_generator
self.stop_list = stop_words
self.dictionary = Dictionary(document_generator())
self.tfidf_model = TfidfModel(self.dictionary)
stop_ids = [self.dictionary.token_to_id[stop_word] for stop_word in self.stop_list
if stop_word in self.dictionary.token_to_id]
once_ids = [token_id for token_id, doc_freq in self.dictionary.doc_freqs.iteritems() if doc_freq == 1]
self.dictionary.filter_tokens(stop_ids + once_ids)
def add_documents(self, documents):
self.dictionary.add_documents(documents)
def __iter__(self):
for document in self.document_generator():
# yield self.dictionary.doc_to_bag_of_words(tokens)
#yield doc_to_vec(len(self.dictionary.items()), self.dictionary.doc_to_bag_of_words(document))
#yield doc_to_vec(len(self.dictionary.items()),
# self.tfidf_model[self.dictionary.doc_to_bag_of_words(document)])
converted_document = self.dictionary.doc_to_bag_of_words(document)
converted_document = self.tfidf_model[converted_document]
word_count = len(self.dictionary.items())
for word_id in xrange(word_count):
if word_id in converted_document:
yield document.id, word_id, converted_document[word_id]
else:
yield document.id, word_id, 0