Python classifier.Classifier类代码示例

    def __init__(self, fname, *args, **kargs):
        Classifier.__init__(self, fname, *args, **kargs)

        # sometimes a threshold value is trained during Bayesian
        # classification to avoid classifying too many 'documents' as
        # one kind or the other
        self.thresholds = [1.0, 1.0]

def main():
    parser = argparse.ArgumentParser(description='Clasificador de musica.\nToma los datos de entrenamiento de un archivo y utiliza algoritmos evolutivos para crear y mejorar las reglas de clasificación.')
    parser.add_argument('-d', '--data', help='Archivo donde se encuentra la información fuente para el clasificador.')
    args = vars(parser.parse_args())

    """
    Los valores default son:
        tamaño discretizacion - 100
        poblacion de generacion - 10
        min fitness para terminar - 0.9
        numero a seleccionar - 4
        porcentaje de mutacion - 0.05
        maximo de generaciones - 10000
        tipo de seleccion - ROULETTE_WHEEL_SELECTION
    """
    defaults = [100, 10, 0.9, 4, 0.05, 10000, selection.ROULETTE_WHEEL_SELECTION]

    classifier = Classifier(args['data'], discrete_intervals=defaults[0], size_rule_generation=defaults[1], filter_list=["skewness", "spectral_rolloff", "energy", "sv", "spread", "centroid", "obsi", "kurtosis"], log_results=True)
    start = time.clock()
    best_results = classifier.train(req_min_fitness=defaults[2], gen_select=defaults[3], mutation_prob=defaults[4], limit_generations=defaults[5])
    duration = (time.clock() - start)*1000
    print "Duration\t", duration, "ms"
    print "Training endend."
    print "Best results:", ', '.join([str(key) + " fitness: " + str(value['fitness']) for key, value in best_results.items()])
    print "Testing:"
    classifier.test()
    print "Testing ended."

def main():
	me=Classifier()
	feature_counter=Counter()
	feature_set=pickle.load(open('validation_set.pkl', 'rb'))
	feature_set_labels=[]
	for tweet, rating in feature_set:
		print rating
		try:
			float(rating)
		except:
			continue
		if float(rating)>0:
			label='positive'
		elif float(rating)<0:
			label='negative'
		else:
			label='neutral'
		feature_set_labels.append((tweet, label))
	feature_list=chain.from_iterable([word_tokenize(process_tweet(tweet)) for tweet, sentiment in feature_set_labels])
	for feat in feature_list:
		feature_counter[feat]+=1
	me.feature_list=[feat for feat, count in feature_counter.most_common(1000)]
	ts=[(me.extract_features(tweet), label) for tweet, label in feature_set]
	print 'training Maxent'
	me.classifier=MaxentClassifier.train(ts)
	return me

def main():
  dbinfo = recover()
  conn = MySQLdb.connect(**dbinfo)

  cur = conn.cursor()

  #Learn
  sql = "SELECT id,article_text,trainpos,trainneg,trainneutral FROM articles WHERE trainset=1 AND (trainpos>0 OR trainneg>0 OR trainneutral>0)"
  cur.execute(sql)
  a = Learner()
  for aid,article_text,trainpos,trainneg,trainneutral in cur.fetchall():
    aid = int(aid)
    items = [ (1, int(trainpos)),(0, int(trainneutral)),(-1, int(trainneg)) ]
    classification = max(items, key=lambda x : x[1])[0]
    a.add_string(article_text, classification)
  a.train()

  #Predict
  sql = "SELECT id,article_text FROM articles"
  cur.execute(sql)
  b = Classifier(a)
  for aid,article_text in cur.fetchall():
    aid = int(aid)
    classification = b.classify(article_text)
    sql = "UPDATE articles SET score=%s WHERE id=%s"
    args = [classification,aid]
    cur.execute(sql,args)
    print aid,classification

  conn.commit()

def GetNewArticles(request):
    # Get the articles from RSS
    # aggregator = NewsAggregator()
    # list_of_articles = aggregator.feedreader()
    classifier = Classifier("filename.pkl")
    # Predict
    list_of_classes = []
    # with open("articles_dump", "wb") as dump:
    #     pickle.dump(list_of_articles, dump, pickle.HIGHEST_PROTOCOL)
    with open("articles_dump") as dump:
        list_of_articles = pickle.load(dump)
    for article in list_of_articles:
        list_of_classes.append(article["content"])
    # print list_of_classes
    res = classifier.predict(np.asarray(list_of_classes))

    for i in range(0, len(list_of_articles)):
        if res[i] == 1:
            cat = "Sports"
        elif res[i] == 2:
            cat = "Economy_business_finance"
        elif res[i] == 3:
            cat = "Science_technology"
        else:
            cat = "Lifestyle_leisure"
        element = list_of_articles[i]
        list_of_articles[i]["category"] = cat
        article = Article(article_title=element["title"], article_content=element["content"], article_category=cat)
        article.save()
    json_object = json.dumps(list_of_articles)
    return HttpResponse(json_object)

def create_predict(HudongItem_csv):
	# 读取neo4j内容 
	db = Neo4j()
	db.connectDB()
	data_set = db.getLabeledHudongItem('labels.txt')
	classifier = Classifier('wiki.zh.bin')
	classifier.load_trainSet(data_set)     
	classifier.set_parameter(weight=[1.0, 3.0, 0.2, 4.0, 0],k=10)
	predict_List = readCSVbyColumn(HudongItem_csv, 'title')
	file_object = open('predict_labels2.txt','a')
	
	count = 0
	vis = set()
	for p in predict_List:
		cur = HudongItem(db.matchHudongItembyTitle(p))
		count += 1
		title = cur.title
		if title in vis:
			continue
		vis.add(title)
		label = classifier.KNN_predict(cur)
		print(str(title)+" "+str(label)+": "+str(count)+"/"+str(len(predict_List)))
		file_object.write(str(title)+" "+str(label)+"\n")
		
	file_object.close()
	
#create_predict('hudong_pedia2.csv')
	

def eval_classifier(classifierToUse, featuresToUse, testOrTrain="train"):

    print("Chosen feature: {0}".format(featuresToUse) )
    print("Chosen classifier: {0}".format(classifierToUse))

    fe = FeatureExtractor(featuresToUse)
    dataset = DataSet(fe)
    classifier = Classifier()
    evaluate = Evaluation()

    print "test or Train %s" % testOrTrain
    for feature_class, files in getTestData(testOrTrain).items():
        print "%s" % testOrTrain
        for f in files:
            dataset.addFile(feature_class, f)

    print "Dataset initialized"
    print_class_stats(dataset.classes)

    print "Test set created."
    a_train, a_test, c_train, c_test = train_test_split(dataset.featureVector, dataset.classes, test_size=0.9)
    
    c_pred = classifier.classification(a_train,a_test,c_train,c_test,classifierToUse)
    
    evaluate.evaluate(c_pred,c_test,featuresToUse,classifierToUse)

  def __init__(self, D, H, W, K, iternum):
    Classifier.__init__(self, D, H, W, K, iternum)
    self.L = 100 # size of hidden layer

    """ Layer 1 Parameters """
    # weight matrix: [M * L]
    self.A1 = 0.01 * np.random.randn(self.M, self.L)
    # bias: [1 * L]
    self.b1 = np.zeros((1,self.L))

    """ Layer 3 Parameters """
    # weight matrix: [L * K]
    self.A3 = 0.01 * np.random.randn(self.L, K)
    # bias: [1 * K]
    self.b3 = np.zeros((1,K))

    """ Hyperparams """
    # learning rate
    self.rho = 1e-2
    # momentum
    self.mu = 0.9
    # reg strencth
    self.lam = 0.1
    # velocity for A1: [M * L]
    self.v1 = np.zeros((self.M, self.L))
    # velocity for A3: [L * K] 
    self.v3 = np.zeros((self.L, K))
    return

def runNeuralNetwork(train, test, batchSize, classNum, hLayer=None, mode=None, momentumFactor=0.0):
    """
    A function that call the the classifier to train a learning model.
    Args:
        train: training examples (numpy)
        test: testing examples (numpy)
        batchSize: the number of training example for each iteration
        classNum: the number of classes
        hLayer: number of the hidden layer nodes (list)
        mode: weight initializing mode
        momentumFactor: momentum factor
    """
    print ""
    print "Neural Network =============================="
    print " - number of hidden layer nodes:",
    if hLayer is not None:
        print hLayer
    else:
        print " default (one hidden layer with node number = 2 * feature number)"

    print " - weight initialization mode:",
    if mode is not None:
        print mode
    else:
        print "default"

    print " - momentum factor", momentumFactor

    nn = Classifier("neural_network", hidden_layer=hLayer, weightInitMode=mode, momentumFactor=momentumFactor)
    nn.train(train, test, classNum, batchSize)
    nn.test(test, "test")

def build_model_mnist():

    # CNN
    filter_size = (5, 5)
    activation = Rectifier().apply
    pooling_size = (2, 2)
    num_filters = 50
    layer0 = ConvolutionalLayer(activation=activation, filter_size=filter_size, num_filters=num_filters,
                              pooling_size=pooling_size,
                              weights_init=Uniform(width=0.1),
                              biases_init=Uniform(width=0.01), name="layer_0")

    filter_size = (3, 3)
    activation = Rectifier().apply
    num_filters = 20
    layer1 = ConvolutionalLayer(activation=activation, filter_size=filter_size, num_filters=num_filters,
                              pooling_size=pooling_size,
                              weights_init=Uniform(width=0.1),
                              biases_init=Uniform(width=0.01), name="layer_1")

    conv_layers = [layer0, layer1]
    convnet = ConvolutionalSequence(conv_layers, num_channels= 1,
                                    image_size=(28, 28))

    convnet.initialize()
    output_dim = np.prod(convnet.get_dim('output'))
    mlp = MLP(activations=[Identity()], dims=[output_dim, 10],
                        weights_init=Uniform(width=0.1),
                        biases_init=Uniform(width=0.01), name="layer_2")
    mlp.initialize()

    classifier = Classifier(convnet, mlp)
    classifier.initialize()
    return classifier

    def run(self):
        """
        Function: Run
        -------------
        This function will evaluate your solution! You do not need to
        write any code in this file, however you SHOULD understand this
        function!
        """
        print "Running the full pipeline!"
        K=25
        trainImages = util.loadTrainImages()[:1000]
        testImages = util.loadTestImages()

        classifier = Classifier()

        print 'Training..........'
        classifier.train(trainImages, K)

        trainPredictions = classifier.test(trainImages)
        trainAccuracy = self.evaluate(trainPredictions, trainImages)

        print 'Testing...........'
        testPredictions = classifier.test(testImages)
        testAccuracy = self.evaluate(testPredictions, testImages)

        print 'All done. Here is your summary:'
        self.reportAccuracy(trainAccuracy, 'Train Accuracy')
        self.reportAccuracy(testAccuracy, 'Test Accuracy')

def average_multiple_runs(num_runs, options, args):
    for num, option in enumerate(options):
        print "Running", num_runs, "iterations with options:", option
        list_best_results = []
        list_test_results = []
        list_correct_results = []
        for i in range(num_runs):
            print "Running #" + str(i + 1)
            classifier = Classifier(args['data'], discrete_intervals=option[0], size_rule_generation=option[1], filter_list=["skewness", "spectral_rolloff", "energy", "sv", "spread", "centroid", "obsi", "kurtosis"], log_results=False)
            best_results = classifier.train(req_min_fitness=option[2], gen_select=option[3], mutation_prob=option[4], limit_generations=option[5], selection_type=option[6])
            test_results, correct_results = classifier.test()
            list_best_results.append(best_results)
            list_test_results.append(test_results)
            list_correct_results.append(correct_results)
        print "Results for option: ", option
        print "run\ttype\tgen\tfitness"
        for i, results in enumerate(list_best_results):
            for rule, result in results.items():
                print str(i + 1) + "\t" + rule[:7] + "\t" + str(result["generation"]) + "\t" + str(result["fitness"])

        print "run\ttype\tavg correct rules"
        for i, results in enumerate(list_test_results):
            for avg_map in results:
                print str(i + 1) + "\t" + avg_map.keys()[0][:7] + "\t" + str(avg_map[avg_map.keys()[0]])

        print "run\ttype\tavg correct results"
        for i, results in enumerate(list_correct_results):
            for avg_map in results:
                print str(i + 1) + "\t" + avg_map.keys()[0][:7] + "\t" + str(avg_map[avg_map.keys()[0]])

def test_classify_by_randomforest():
    stock_d = testdata()
    ti = TechnicalIndicators(stock_d)

    filename = 'test_N225_randomforest.pickle'
    clffile = os.path.join(os.path.dirname(
                           os.path.abspath(__file__)),
                           '..', 'clf',
                           filename)

    if os.path.exists(clffile):
        os.remove(clffile)

    clf = Classifier(filename)
    ti.calc_ret_index()
    ret = ti.stock['ret_index']

    train_X, train_y = clf.train(ret, classifier="Random Forest")

    eq_(filename, os.path.basename(clf.filename))

    r = round(train_X[-1][-1], 5)
    expected = 1.35486
    eq_(r, expected)

    r = round(train_X[0][0], 5)
    expected = 1.08871
    eq_(r, expected)

    expected = 14
    r = len(train_X[0])
    eq_(r, expected)

    expected = 120
    r = len(train_X)
    eq_(r, expected)

    expected = [1, 0, 0, 0, 1, 1, 0, 0, 0, 0,
                0, 0, 1, 0, 0, 1, 0, 1, 0, 1,
                1, 0, 1, 1, 1, 1, 1, 0, 1, 0,
                1, 1, 1, 1, 0, 1, 0, 1, 1, 0,
                1, 0, 0, 1, 1, 1, 1, 1, 1, 1,
                0, 0, 0, 1, 0, 0, 1, 1, 1, 1,
                1, 0, 1, 0, 0, 0, 0, 0, 0, 1,
                1, 1, 0, 0, 1, 0, 1, 1, 0, 1,
                1, 0, 1, 1, 0, 1, 0, 0, 1, 0,
                1, 1, 0, 0, 1, 0, 1, 0, 1, 1,
                1, 1, 1, 0, 1, 1, 1, 0, 0, 1,
                1, 0, 0, 1, 1, 1, 0, 1, 1, 0]

    for r, e in zip(train_y, expected):
        eq_(r, e)

    expected = 1
    test_y = clf.classify(ret)
    assert(test_y[0] == 0 or test_y[0] == 1)

    if os.path.exists(clffile):
        os.remove(clffile)

def main(mode='test'):
    cl = Classifier()
    cl.create_db('bunyk.db')

    if mode == 'test':
        test(cl)
    else:
        train(cl, 'http://bunyk.wordpress.com')

    def setUp(self):
        text = u"Comment Google classe les pages Internet"

        c = Classifier(CleanTextUtil("french"))
        c.add_text(text)

        self.dictionary_db = c.dictionary_db
        self.vi = VectorItem("googl", "1")