本文整理汇总了Python中pyspark.mllib.feature.Word2Vec方法的典型用法代码示例。如果您正苦于以下问题：Python feature.Word2Vec方法的具体用法？Python feature.Word2Vec怎么用？Python feature.Word2Vec使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspark.mllib.feature的用法示例。
在下文中一共展示了feature.Word2Vec方法的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_word2vec_setters
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def test_word2vec_setters(self):
model = Word2Vec() \
.setVectorSize(2) \
.setLearningRate(0.01) \
.setNumPartitions(2) \
.setNumIterations(10) \
.setSeed(1024) \
.setMinCount(3) \
.setWindowSize(6)
self.assertEqual(model.vectorSize, 2)
self.assertTrue(model.learningRate < 0.02)
self.assertEqual(model.numPartitions, 2)
self.assertEqual(model.numIterations, 10)
self.assertEqual(model.seed, 1024)
self.assertEqual(model.minCount, 3)
self.assertEqual(model.windowSize, 6) 示例2: create_model_text
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def create_model_text(self, data, params):
learningRate = float(params.get('learningRate', 0.025))
numIterations = int(params.get('numIterations', 10))
minCount = int(params.get('minCount', 5))
word2vec = Word2Vec()
word2vec.setLearningRate(learningRate)
word2vec.setNumIterations(numIterations)
word2vec.setMinCount(minCount)
inp = data.map(lambda row: row.split(" "))
return word2vec.fit(inp) 示例3: main
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def main(in_loc, out_dir):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO)
sc = ps.SparkContext(appName='Word2Vec')
logger.info('Distributing input data')
raw_data = sc.textFile(in_loc).cache()
data = raw_data.map(lambda line: line.split(' '))
print(data.getNumPartitions())
logger.info('Training Word2Vec model')
model = Word2Vec().setVectorSize(128).setNumIterations(5).fit(data)
w2v_dict = model.getVectors()
logger.info('Saving word to vectors dictionary')
with open(path.join(out_dir, 'w2v_dict.pkl'), 'wb') as f:
cPickle.dump(w2v_dict, f, cPickle.HIGHEST_PROTOCOL)
model.save(sc, out_dir) 示例4: init_model_controller
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def init_model_controller(self):
model_type = self.job_args['model']['type']
if model_type == 'KMeans':
self.controller = KMeansModelController()
elif model_type == 'Recommendation':
self.controller = RecommendationController()
elif model_type == 'LogisticRegression':
self.controller = LogisticRegressionModelController()
elif model_type == 'LinearRegression':
self.controller = LinearRegressionModelController()
elif model_type == 'RidgeRegression':
self.controller = RidgeRegressionModelController()
elif model_type == 'DecisionTreeRegression':
self.controller = DecisionTreeModelController('Regression')
elif model_type == 'DecisionTreeClassification':
self.controller = DecisionTreeModelController('Classification')
elif model_type == 'RandomForestRegression':
self.controller = RandomForestModelController('Regression')
elif model_type == 'RandomForestClassification':
self.controller = RandomForestModelController('Classification')
elif model_type == 'Word2Vec':
self.controller = Word2VecModelController()
elif model_type == 'FPGrowth':
self.controller = FPGrowthModelController()
elif model_type == 'NaiveBayes':
self.controller = NaiveBayesModelController() 示例5: test_word2vec_get_vectors
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def test_word2vec_get_vectors(self):
data = [
["a", "b", "c", "d", "e", "f", "g"],
["a", "b", "c", "d", "e", "f"],
["a", "b", "c", "d", "e"],
["a", "b", "c", "d"],
["a", "b", "c"],
["a", "b"],
["a"]
]
model = Word2Vec().fit(self.sc.parallelize(data))
self.assertEqual(len(model.getVectors()), 3) 示例6: generate_word2vec_model
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def generate_word2vec_model(doc):
return Word2Vec().setVectorSize(10).setSeed(410).fit(doc) 示例7: test_word2vec_setters
# 需要导入模块: from pyspark.mllib import feature [as 别名]
# 或者: from pyspark.mllib.feature import Word2Vec [as 别名]
def test_word2vec_setters(self):
model = Word2Vec() \
.setVectorSize(2) \
.setLearningRate(0.01) \
.setNumPartitions(2) \
.setNumIterations(10) \
.setSeed(1024) \
.setMinCount(3)
self.assertEqual(model.vectorSize, 2)
self.assertTrue(model.learningRate < 0.02)
self.assertEqual(model.numPartitions, 2)
self.assertEqual(model.numIterations, 10)
self.assertEqual(model.seed, 1024)
self.assertEqual(model.minCount, 3)