本文整理汇总了Python中pyspark.mllib.tree.DecisionTree.trainClassifier方法的典型用法代码示例。如果您正苦于以下问题:Python DecisionTree.trainClassifier方法的具体用法?Python DecisionTree.trainClassifier怎么用?Python DecisionTree.trainClassifier使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspark.mllib.tree.DecisionTree
的用法示例。
在下文中一共展示了DecisionTree.trainClassifier方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: main
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def main():
sc = SparkContext(appName="MyApp")
sc.setLogLevel('ERROR')
# Parse data
train_labels, train_data = load_data('train.csv')
dummy_labels, test_data = load_data('test.csv', use_labels=False)
# Map each data point's label to its features
train_set = reformatData(train_data, train_labels)
test_set = reformatData(test_data, dummy_labels)
# Parallelize the data
parallelized_train_set = sc.parallelize(train_set)
parallelized_test_set = sc.parallelize(test_set)
# Split the data
trainSet, validationSet = parallelized_train_set.randomSplit([1.0, 0.0], seed=42)
# Train the models
decisionTreeModel = DecisionTree.trainClassifier(trainSet, numClasses=5, categoricalFeaturesInfo={},
impurity='gini', maxBins=55, maxDepth=30, minInstancesPerNode=2)
# Test the model
testDecisionTree(decisionTreeModel, parallelized_test_set)
示例2: generateDecisionTree
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def generateDecisionTree():
if os.path.exists(DT_PATH):
print("DT_PATH Already available")
return
global model
data = sc.textFile(F_PATH).map(parseLine)
(trainingData, testData) = data.randomSplit([0.9, 0.1], seed=1L)
model = DecisionTree.trainClassifier(trainingData, numClasses=classes.__len__(), categoricalFeaturesInfo={},
impurity='gini', maxDepth=5, maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(testData.count())
print('Test Error = ', str(testErr))
print('Learned classification tree model:')
print(model.toDebugString())
modelStatistics(labelsAndPredictions)
# Save and load model
model.save(sc, DT_PATH)
print("Decision Tree model saved!")
示例3: trainClassifier
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def trainClassifier(self):
# get the current time
current = time()
# get the tags
tags = self.tags
numeric = self.numeric
x = self.x
y = self.y
# get the training data
training_data = self.training_labeled
# start training the tree model
self.tree_model = DecisionTree.trainClassifier(
training_data,
numClasses=4,
categoricalFeaturesInfo={0 : len(tags), 1 : len(numeric), 2 : len(x), 3 : len(y)},
impurity="gini",
maxDepth=5,
maxBins=1000)
print self.tree_model
# total time
total = time() - current
print "Classifier trained in {} seconds.".format(round(total, 3))
# start evaluating the model
self.evaluate()
示例4: decisionTree
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def decisionTree(trainingRDD, trainingRDDHashed, testRDDHashed, testRDD):
# Get size of RDD
nFilesV = trainingRDDHashed.count()
nFilesT = testRDDHashed.count()
# Train the Decision Tree Model
trainedModel = DecisionTree.trainClassifier(
trainingRDD,
numClasses=2,
categoricalFeaturesInfo={},
impurity='gini',
maxDepth=2,
maxBins=3)
# Test the Model on the Training Set
predictions = trainedModel.predict(trainingRDD.map(lambda x: x.features))
labelsAndPredictions = trainingRDD.map(
lambda lp: lp.label).zip(predictions).countByValue()
# Map to Dictionary for obtaining Results
resultsValidation = defaultdict(lambda: 0, labelsAndPredictions)
nFilesV = trainingRDDHashed.count()
nFilesT = testRDDHashed.count()
# Get F-Score and Accuracy Value
AccuracyV, fScoreV = getAccuracy(resultsValidation, nFilesV)
# Test the Model on the Test Set
predictions = trainedModel.predict(testRDD.map(lambda x: x.features))
labelsAndPredictions = testRDD.map(
lambda lp: lp.label).zip(predictions).countByValue()
# Map to Dictionary for obtaining Results
resultsTest = defaultdict(lambda: 0, labelsAndPredictions)
AccuracyT, fScoreT = getAccuracy(resultsTest, nFilesT)
# Print Results
print(' Results for Decision Tree')
print(' Training Set: %.3f and F-Score: %.3f') % (AccuracyV, fScoreV)
print(' Test Set: %.3f and F-Score: %.3f') % (AccuracyT, fScoreT)
# Return the Result List
return AccuracyV, fScoreV, AccuracyT, fScoreT
示例5: trainModel
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def trainModel(trainingData):
print '\nTraining Decision Tree model started'
Utils.logTime()
model = DecisionTree.trainClassifier(trainingData, numClasses=2, categoricalFeaturesInfo={}, impurity='gini', maxDepth=5,maxBins=32)
print '\nTraining Decision Tree model finished'
Utils.logTime()
return model
示例6: RunDecisionTree
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def RunDecisionTree(tf):
rdd = tf.map(parseAsLabeledPoints)
train, test = rdd.randomSplit([.8, .2])
model = DecisionTree.trainClassifier(train, numClasses=numCat, categoricalFeaturesInfo={},impurity='gini', maxDepth=5, maxBins=100)
predictions = model.predict(train.map(lambda x: x.features))
labelsAndPredictions = train.map(lambda lp: lp.label).zip(predictions)
trainErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(test.count())
print('Training Error = ' + str(trainErr))
示例7: train
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def train(self, num_classes=2, categorical_features=None, max_depth=5):
categorical_features = categorical_features or {}
model = DecisionTree.trainClassifier(
self._labeled_feature_vector_rdd(),
numClasses=num_classes,
categoricalFeaturesInfo=categorical_features,
maxDepth=max_depth)
return DecisionTreeModel(model, self.feature_cols)
示例8: test_classification
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def test_classification(self):
from pyspark.mllib.classification import LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes
from pyspark.mllib.tree import DecisionTree, RandomForest, GradientBoostedTrees
data = [
LabeledPoint(0.0, [1, 0, 0]),
LabeledPoint(1.0, [0, 1, 1]),
LabeledPoint(0.0, [2, 0, 0]),
LabeledPoint(1.0, [0, 2, 1])
]
rdd = self.sc.parallelize(data)
features = [p.features.tolist() for p in data]
lr_model = LogisticRegressionWithSGD.train(rdd)
self.assertTrue(lr_model.predict(features[0]) <= 0)
self.assertTrue(lr_model.predict(features[1]) > 0)
self.assertTrue(lr_model.predict(features[2]) <= 0)
self.assertTrue(lr_model.predict(features[3]) > 0)
svm_model = SVMWithSGD.train(rdd)
self.assertTrue(svm_model.predict(features[0]) <= 0)
self.assertTrue(svm_model.predict(features[1]) > 0)
self.assertTrue(svm_model.predict(features[2]) <= 0)
self.assertTrue(svm_model.predict(features[3]) > 0)
nb_model = NaiveBayes.train(rdd)
self.assertTrue(nb_model.predict(features[0]) <= 0)
self.assertTrue(nb_model.predict(features[1]) > 0)
self.assertTrue(nb_model.predict(features[2]) <= 0)
self.assertTrue(nb_model.predict(features[3]) > 0)
categoricalFeaturesInfo = {0: 3} # feature 0 has 3 categories
dt_model = DecisionTree.trainClassifier(
rdd, numClasses=2, categoricalFeaturesInfo=categoricalFeaturesInfo)
self.assertTrue(dt_model.predict(features[0]) <= 0)
self.assertTrue(dt_model.predict(features[1]) > 0)
self.assertTrue(dt_model.predict(features[2]) <= 0)
self.assertTrue(dt_model.predict(features[3]) > 0)
rf_model = RandomForest.trainClassifier(
rdd, numClasses=2, categoricalFeaturesInfo=categoricalFeaturesInfo, numTrees=100)
self.assertTrue(rf_model.predict(features[0]) <= 0)
self.assertTrue(rf_model.predict(features[1]) > 0)
self.assertTrue(rf_model.predict(features[2]) <= 0)
self.assertTrue(rf_model.predict(features[3]) > 0)
gbt_model = GradientBoostedTrees.trainClassifier(
rdd, categoricalFeaturesInfo=categoricalFeaturesInfo)
self.assertTrue(gbt_model.predict(features[0]) <= 0)
self.assertTrue(gbt_model.predict(features[1]) > 0)
self.assertTrue(gbt_model.predict(features[2]) <= 0)
self.assertTrue(gbt_model.predict(features[3]) > 0)
示例9: DecisionTreeProcess
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def DecisionTreeProcess(trainingSet, testSet, imp, dtMaxDepth, dtMaxBins):
decisionTreeModel = DecisionTree.trainClassifier(trainingSet, numClasses = 4,categoricalFeaturesInfo={},
impurity=imp,maxDepth=dtMaxDepth, maxBins=dtMaxBins)
predictions = decisionTreeModel.predict(trainingSet.map(lambda item: item.features))
trainingLabelsAndPredictions = trainingSet.map(lambda item: item.label).zip(predictions)
eva.calculateErrorRate("\nClassification model Training set", trainingLabelsAndPredictions)
predictions = decisionTreeModel.predict(testSet.map(lambda item: item.features))
testLabelsAndPredictions = testSet.map(lambda item: item.label).zip(predictions)
eva.calculateErrorRate("\nClassification model Test set", testLabelsAndPredictions)
return decisionTreeModel
示例10: classify
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def classify(sc, sample):
def ff(x):
newsample = []
nl = ["rainy","sad","lack"]
ml = ["cloudy","soso","enough"]
pl = ["sunny","happy","most"]
for i in x:
if i in nl:
newsample.append(0)
elif i in ml:
newsample.append(1)
elif i in pl:
newsample.append(2)
return newsample
f = lambda x:1 if x=="yes" else 0
traindata = sc.parallelize(sample).map(lambda x:(ff(x[0]),f(x[1])))
traindata = traindata.map(lambda x:LabeledPoint(x[1],x[0]))
testdata = traindata.first()
print testdata
######
# print "logistic"
# lrModel = LogisticRegressionWithSGD.train(traindata, 10)
# prediction = lrModel.predict(testdata.features)
# print prediction
#####
# print "svm"
# svmModel = SVMWithSGD.train(traindata, 10)
# prediction = svmModel.predict(testdata.features)
# print prediction
#
#
# ####
# print "naive bayes"
# nbModel = NaiveBayes.train(traindata)
# prediction = nbModel.predict(testdata.features)
# print prediction
#
#
# ####
print "decesion tree"
detreeModel = DecisionTree.trainClassifier(traindata, 2, {})
prediction = detreeModel.predict(testdata.features)
print prediction
示例11: main
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def main(input_file):
sc = pyspark.SparkContext(appName="DecisionTree")
data = MLUtils.loadLabeledPoints(sc, input_file)
trainingData, testData = data.randomSplit([0.70, 0.3])
# Cache in memory for faster training
trainingData.cache()
model = DecisionTree.trainClassifier(trainingData, numClasses=4, impurity='gini',
categoricalFeaturesInfo={}, maxDepth=16, maxBins=10)
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(testData.count())
# print tree_model.toDebugString()
print ""
print ""
print "Test Erros: {}".format(round(testErr,4))
示例12: trainModel
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def trainModel(self, vectSpace, path):
try:
if self.type == 'NaiveBayes':
model = NaiveBayes.train(vectSpace)
elif self.type == 'DecisionTree':
model = DecisionTree.trainClassifier(vectSpace, numClasses = len(self.category), categoricalFeaturesInfo={}, impurity='gini', maxDepth=5, maxBins=5)
if not os.path.exists(path):
os.makedirs(path)
else:
shutil.rmtree(path)
os.makedirs(path)
model.save(self.sc, path)
except:
print "Unexpected error:", sys.exc_info()[0]
raise
return model
示例13: process
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def process(sc, dtClusterNum, dtMaxDepth, dtMaxBins, eigenVecFile, markedClusterFile):
filteredEigenVec = sc.textFile(eigenVecFile).map(lambda item: removeVirtualPart(item)).collect()
clusterIDs = sc.textFile(markedClusterFile).map(lambda item: extractClusterID(item)).collect()
clusterIdEigenVecMapRDD = sc.parallelize(clusterIDs).zip(sc.parallelize(filteredEigenVec))
labeledClusterIdEigenVecMapRdd = clusterIdEigenVecMapRDD.map(lambda item: LabeledPoint(item[0], item[1]))
trainingSet, testSet = labeledClusterIdEigenVecMapRdd.randomSplit([0.7, 0.3])
decisionTreeModel = DecisionTree.trainClassifier(trainingSet, numClasses = dtClusterNum,
categoricalFeaturesInfo={},impurity='entropy',maxDepth=dtMaxDepth, maxBins=dtMaxBins)
predictions = decisionTreeModel.predict(trainingSet.map(lambda item: item.features))
trainingLabelsAndPredictions = trainingSet.map(lambda item: item.label).zip(predictions)
eva.calculateErrorRate("\nCluster model Training set", trainingLabelsAndPredictions)
predictions = decisionTreeModel.predict(testSet.map(lambda item: item.features))
testLabelsAndPredictions = testSet.map(lambda item: item.label).zip(predictions)
eva.calculateErrorRate("\nCluster model Test set", testLabelsAndPredictions)
return decisionTreeModel
示例14: create_model
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def create_model(name, training):
if name == 'logistic':
print_box()
print "Logistic Regression Model"
print_box()
model = LogisticRegressionWithLBFGS.train(training)
elif name == 'tree':
print_box()
print "Decision Tree Model"
print_box()
model = DecisionTree.trainClassifier(training, numClasses=2, categoricalFeaturesInfo={},
impurity='gini', maxDepth=5, maxBins=32)
elif name == 'rf':
print_box()
print "Random Forest Model"
print_box()
model = RandomForest.trainClassifier(training, numClasses=2, categoricalFeaturesInfo={},
numTrees=15, featureSubsetStrategy="auto", impurity='gini', maxDepth=5, maxBins=50)
return model
示例15: trainOptimalModel
# 需要导入模块: from pyspark.mllib.tree import DecisionTree [as 别名]
# 或者: from pyspark.mllib.tree.DecisionTree import trainClassifier [as 别名]
def trainOptimalModel(trainingData, testData):
print "\nTraining optimal Decision Tree model started!"
Utils.logTime()
impurityVals = ['gini', 'entropy']
maxDepthVals = [3,4,5,6,7]
maxBinsVals = [8,16,32]
optimalModel = None
optimalMaxDepth = None
optimalImpurity = None
optimalBinsVal = None
minError = None
try:
for curImpurity in impurityVals:
for curMaxDepth in maxDepthVals:
for curMaxBins in maxBinsVals:
model = DecisionTree.trainClassifier(trainingData,
numClasses=2,
categoricalFeaturesInfo={},
impurity=curImpurity,
maxDepth=curMaxDepth,
maxBins=curMaxBins)
testErr, PR, ROC = Evaluation.evaluate(model, testData)
if testErr < minError or not minError:
minError = testErr
optimalImpurity = curImpurity
optimalMaxDepth = curMaxDepth
optimalBinsVal = curMaxBins
optimalModel = model
except:
msg = "\nException during model training with below parameters:"
msg += "\timpurity: " + str(curImpurity)
msg += "\tmaxDepth: " + str(curMaxDepth)
msg += "\tmaxBins: " + str(curMaxBins)
Utils.logMessage(msg)
logMessage(optimalModel, optimalMaxDepth, optimalImpurity, optimalBinsVal, minError)
return optimalModel