本文整理匯總了Python中sklearn.feature_extraction.text.TfidfVectorizer.vocabulary_方法的典型用法代碼示例。如果您正苦於以下問題：Python TfidfVectorizer.vocabulary_方法的具體用法？Python TfidfVectorizer.vocabulary_怎麽用？Python TfidfVectorizer.vocabulary_使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類sklearn.feature_extraction.text.TfidfVectorizer的用法示例。
在下文中一共展示了TfidfVectorizer.vocabulary_方法的2個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: load_tfidf
# 需要導入模塊: from sklearn.feature_extraction.text import TfidfVectorizer [as 別名]
# 或者: from sklearn.feature_extraction.text.TfidfVectorizer import vocabulary_ [as 別名]
def load_tfidf(vocab_path, idf_weights_path):
"""Loads tfidf vectorizer from its components.
:param str vocab_path: path to the vectorizer vocabulary JSON.
:param str idf_weights_path: path to idf weights JSON.
:rtype: sklearn.feature_extraction.text.TfidfVectorizer
"""
tfidf = TfidfVectorizer(analyzer=lambda x: x,
vocabulary=json.load(open(vocab_path)))
idf_vector = np.array(json.load(open(idf_weights_path)))
tfidf._tfidf._idf_diag = scipy.sparse.diags([idf_vector], [0])
tfidf.vocabulary_ = tfidf.vocabulary
return tfidf示例2: train_tfidf
# 需要導入模塊: from sklearn.feature_extraction.text import TfidfVectorizer [as 別名]
# 或者: from sklearn.feature_extraction.text.TfidfVectorizer import vocabulary_ [as 別名]
def train_tfidf(self, tokenizer='custom', corpus='news'):
if tokenizer == 'custom':
tokenizer = self.tokenize
nltk_corpus = []
if corpus == 'all':
nltk_corpus += [nltk.corpus.gutenberg.raw(f_id) for f_id in nltk.corpus.gutenberg.fileids()]
nltk_corpus += [nltk.corpus.webtext.raw(f_id) for f_id in nltk.corpus.webtext.fileids()]
nltk_corpus += [nltk.corpus.brown.raw(f_id) for f_id in nltk.corpus.brown.fileids()]
nltk_corpus += [nltk.corpus.reuters.raw(f_id) for f_id in nltk.corpus.reuters.fileids()]
elif corpus == 'news':
nltk_corpus += self.get_bbc_news_corpus()
if self.verbose:
print "LENGTH nltk corpus corpus: {}".format(sum([len(d) for d in nltk_corpus]))
vectorizer = TfidfVectorizer(
max_df=1.0,
min_df=2,
encoding='utf-8',
decode_error='strict',
max_features=None,
stop_words='english',
ngram_range=(1, 3),
norm='l2',
tokenizer=tokenizer,
use_idf=True,
sublinear_tf=False)
#vectorizer.fit_transform(nltk_corpus)
vectorizer.fit(nltk_corpus)
# Avoid having to pickle instance methods, we will set this method on on load
vectorizer.tokenizer = None
keys = np.array(vectorizer.vocabulary_.keys(), dtype=str)
values = np.array(vectorizer.vocabulary_.values(), dtype=int)
stop_words = np.array(list(vectorizer.stop_words_), dtype=str)
with tables.openFile(self.data_path + 'tfidf_keys.hdf', 'w') as f:
atom = tables.Atom.from_dtype(keys.dtype)
ds = f.createCArray(f.root, 'keys', atom, keys.shape)
ds[:] = keys
with tables.openFile(self.data_path + 'tfidf_values.hdf', 'w') as f:
atom = tables.Atom.from_dtype(values.dtype)
ds = f.createCArray(f.root, 'values', atom, values.shape)
ds[:] = values
with tables.openFile(self.data_path + 'tfidf_stop_words.hdf', 'w') as f:
atom = tables.Atom.from_dtype(stop_words.dtype)
ds = f.createCArray(f.root, 'stop_words', atom, stop_words.shape)
ds[:] = stop_words
vectorizer.vocabulary_ = None
vectorizer.stop_words_ = None
with open(self.data_path + 'tfidf.pkl', 'wb') as fin:
cPickle.dump(vectorizer, fin)
vectorizer.vocabulary_ = dict(zip(keys, values))
vectorizer.stop_words_ = stop_words
return vectorizer