本文整理汇总了Python中dictionary.Dictionary.add_documents方法的典型用法代码示例。如果您正苦于以下问题：Python Dictionary.add_documents方法的具体用法？Python Dictionary.add_documents怎么用？Python Dictionary.add_documents使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dictionary.Dictionary的用法示例。
在下文中一共展示了Dictionary.add_documents方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TextCorpus
# 需要导入模块: from dictionary import Dictionary [as 别名]
# 或者: from dictionary.Dictionary import add_documents [as 别名]
class TextCorpus(interfaces.CorpusABC):
"""
Helper class to simplify the pipeline of getting bag-of-words vectors (= a
gensim corpus) from plain text.
This is an abstract base class: override the `get_texts()` method to match
your particular input.
Given a filename (or a file-like object) in constructor, the corpus object
will be automatically initialized with a dictionary in `self.dictionary` and
will support the `iter` corpus method. You must only provide a correct `get_texts`
implementation.
"""
def __init__(self, input=None):
super(TextCorpus, self).__init__()
self.input = input
self.dictionary = Dictionary()
if input is not None:
self.dictionary.add_documents(self.get_texts())
else:
logger.warning("No input document stream provided; assuming "
"dictionary will be initialized some other way.")
def __iter__(self):
"""
The function that defines a corpus.
Iterating over the corpus must yield sparse vectors, one for each document.
"""
for text in self.get_texts():
yield self.dictionary.doc2bow(text, allow_update=False)
def getstream(self):
return getstream(self.input)
def get_texts(self):
"""
Iterate over the collection, yielding one document at a time. A document
is a sequence of words (strings) that can be fed into `Dictionary.doc2bow`.
Override this function to match your input (parse input files, do any
text preprocessing, lowercasing, tokenizing etc.). There will be no further
preprocessing of the words coming out of this function.
"""
# Instead of raising NotImplementedError, let's provide a sample implementation:
# assume documents are lines in a single file (one document per line).
# Yield each document as a list of lowercase tokens, via `utils.tokenize`.
length = 0
for lineno, line in enumerate(getstream(self.input)):
length += 1
yield utils.tokenize(line, lowercase=True)
self.length = length
def __len__(self):
return self.length # will throw if corpus not initialized示例2: Corpus
# 需要导入模块: from dictionary import Dictionary [as 别名]
# 或者: from dictionary.Dictionary import add_documents [as 别名]
class Corpus(object):
"""
"""
def __init__(self, document_generator, stop_words):
self.document_generator = document_generator
self.stop_list = stop_words
self.dictionary = Dictionary(document_generator())
self.tfidf_model = TfidfModel(self.dictionary)
stop_ids = [self.dictionary.token_to_id[stop_word] for stop_word in self.stop_list
if stop_word in self.dictionary.token_to_id]
once_ids = [token_id for token_id, doc_freq in self.dictionary.doc_freqs.iteritems() if doc_freq == 1]
self.dictionary.filter_tokens(stop_ids + once_ids)
def add_documents(self, documents):
self.dictionary.add_documents(documents)
def __iter__(self):
for document in self.document_generator():
# yield self.dictionary.doc_to_bag_of_words(tokens)
#yield doc_to_vec(len(self.dictionary.items()), self.dictionary.doc_to_bag_of_words(document))
#yield doc_to_vec(len(self.dictionary.items()),
# self.tfidf_model[self.dictionary.doc_to_bag_of_words(document)])
converted_document = self.dictionary.doc_to_bag_of_words(document)
converted_document = self.tfidf_model[converted_document]
word_count = len(self.dictionary.items())
for word_id in xrange(word_count):
if word_id in converted_document:
yield document.id, word_id, converted_document[word_id]
else:
yield document.id, word_id, 0