Python classifiers.NaiveBayesClassifier类代码示例

def generateIntentionalityClassifier():
    db = dbClient()    
    training = db.training
    cursor = training.find()    
    
    #Reducir la cantidad de registros 
    crs = list(cursor)    
    random.shuffle(crs)
    # split into 90% training and 10% test sets
    p = int(len(crs) * .01)
    cr_test = crs[0:p]        
        
    print "Test", len(cr_test)    
    
    data = []
    t = ""
    for td in cr_test:
        tgram = td["triGram"]
        label = td["label"] 
        #print tgram
        for tg in tgram:
            d = '-'.join(tg)
            t = t + " " + d
        #print t
        data.append((t, label))
        t = ""
    #print data
    cl = NaiveBayesClassifier(data)
    cl.show_informative_features(30)    
    path = "/media/University/UniversityDisc/2-Master/MasterThesis/EjecucionTesis/Desarrollo/PythonProjects/QueryAnalyzer/Models/"
    saveTrainedClassifier(path, cl, "my_classifier_v6.pickle")
    return cl

def create_sentiment():
    """
        Train sentiment model and save.

        Input type: None 
        Output: Model as pickle 
    """

    random.seed(1)

    test = [
        ("The dude presenting Unravel seems like one of the most genuine game developers Ive ever seen I really hope this game works out for him",'pos'),
        ("His hands are shaking Dude looks so stoked and scared at the same time",'pos'),
        ("Right I just felt like I was watching his dream come true It was nice The game looks very well done as well Good for him",'pos'),
        ("Seriously Unravel looks really good actually and honestly seeing him so happy about what hes made is contagious I want to see more of Unravel ",'pos'),
        ("He was so nervous shaking all over his voice quivering",'neg'),
        ("The game looked nice too very cute art style ",'pos'),
        ("You could tell he genuinely wanted to be there it looked like he was even shaking from the excitement  I hope it works out for them aswell",'pos'),
        ("However following that up with the weird PvZ thing was odd To say the least",'neg'),
        ("Haha The game did look nice though Im definitely going to keep an eye on it I enjoy supporting such hopeful developers",'pos'),
        ("Very personable This looks like a buy for me As a dev in a other sector I appreciate this passion",'pos'),
        ("I want to give him a cookie",'pos'),
        ("Im getting a copy Im gonna support my indie devs",'pos'),
        ("The twitch leak was accurate It was like a play by play you start speaking French then switch to English",'neg'),
        ("yep exactly what i was thinking lol its important to note that the twitch leak never had them saying it was Dishonored 2 but that they were honored to be here very different",'neg'),
        ("Honored  Im 100 sure that was intentional",'neg'),
        ("oh yea for sure but wasnt solid enough evidence imo to be like dishonored 2 confirmed just based off that",'neg'),
        ("The confirmation was who was talking not what they were talking about ",'neg'),
        ("How awkward is it for a pop singer to perform at a video game conference",'neg'),
        ("Oh god did they warn him that he will get zero reaction",'neg'),
        ("I really hope so",'pos'),
        ("Almost as bad as Aisha fucking up her dialogue constantly Shes doing alright though E3 is really becoming a mainstream media event Hollywood has nothing like this ComicCon is the only comparison and they dont dazzle it up like E3",'neg')
        ]


    # Grab review data
    reviews = [
        (list(movie_reviews.words(fileid)), category)
        for category in movie_reviews.categories()
        for fileid in movie_reviews.fileids(category)
        ]
    random.shuffle(reviews)

    # Divide into 10% train/test splits
    new_train, new_test = reviews[:1900], reviews[1900:]

    # Train the NB classifier on the train split
    cl = NaiveBayesClassifier(new_train)

    # Compute accuracy
    accuracy = cl.accuracy(test + new_test)
    print("Accuracy: {0}".format(accuracy))

    # Show 5 most informative features
    cl.show_informative_features(5)

    # Save model for use in creating social model sentiment
    with open('sentiment_clf_full.pkl', 'wb') as pk:
        pickle.dump(cl, pk)
    print 'done saving model'

def generateClassifier():
    train = getIntentDataset()

    cl = NaiveBayesClassifier(train)
    cl.show_informative_features(5)    
    path = "/media/University/UniversityDisc/2-Master/MasterThesis/EjecucionTesis/Desarrollo/PythonProjects/QueryAnalyzer/Models/"
    saveTrainedClassifier(path, cl, "intent_classifier_2.pickle")

    def generate_model(self):
        print("Gathering and processing tweets...")
        # Shuffle list of username-label tuples
        tuple_list = usermapping.data_tuples.items()

        # Split and grab tweets for users
        results = utils.flatten([ self.fetch_data(t)
                                  for t in tuple_list ])
         
        # TODO: Cross-validation generation
        trn_ratio = int(len(results) * 0.85)
        shuffle(results)
        print(len(results))
        print(trn_ratio)
        train = results[:trn_ratio]
        test = results[trn_ratio:]

        # Instantiate and train classifier
        print("Training...")
        cl = NaiveBayesClassifier(train)
        cl.train()
        
        # Save model
        print("Saving model...")
        utils.save_model(cl)

        # Classify test
        print("Testing...")
        print("Accuracy: {0}".format(cl.accuracy(test)))
        return cl

def get_analysis(s):

    train = [
        ('I love this sandwich.', 'pos'),
        ('This is an amazing place!', 'pos'),
        ('I feel very good about these beers.', 'pos'),
        ('This is my best work.', 'pos'),
        ("What an awesome view", 'pos'),
        ('I do not like this restaurant', 'neg'),
        ('I am tired of this stuff.', 'neg'),
        ("I can't deal with this", 'neg'),
        ('He is my sworn enemy!', 'neg'),
        ('My boss is horrible.', 'neg')
    ]


    cl = NaiveBayesClassifier(train)

    tweets = Tweet.objects.filter(search_term = s)

    result = []

    for t in tweets:
        d = {}
        c = cl.classify(t.tw_text)
        d['text'] = t.tw_text
        d['res'] = c
        result.append(d)

    return result

 def __init__(self):
     training_data = self._load_data("data")
     self.category_classifier = NaiveBayesClassifier(
         [(x[0], x[1]) for x in training_data])
     self.avoidability_classifier = NaiveBayesClassifier(
         [(x[0], x[2]) for x in training_data])
     self.ordinary_classifier = NaiveBayesClassifier(
         [(x[0], x[3]) for x in training_data])

def train_n_test(file_path):
	documents= load_data(file_path)
	random.shuffle(documents)
	generate_bigrams(data.wordlist)	
	train = documents[0:110]
	test = documents[110:]
	#classifier = NaiveBayesClassifier(train)
	#classifier = NaiveBayesClassifier(train,feature_extractor=get_features)
	classifier = NaiveBayesClassifier(train,feature_extractor=get_feats)
	print classifier.accuracy(test)

class NaiveBayesAnalyzer:
    cl = None

    def __init__(self):
        with open("training_data.json", "r") as f:
            self.cl = NaiveBayesClassifier(f, format="json")
        self.cl.show_informative_features(20)

    def analyze(self, text):
        return self.cl.classify(text)

class LanguageDetector(object):
    def __init__(self, train=SAMPLE_TRAIN, feature_extractor=FeatureExtractors.last_word_extractor()):
        self.train = train
        self.classifier = NaiveBayesClassifier(self.train, feature_extractor)
    
    def accuracy(self, test_set=SAMPLE_TEST):
        return self.classifier.accuracy(test_set)

    def show_features(self):
        return self.classifier.show_informative_features(5)

def main():
    json = raw_input("Where is the json training set?")
    print "Program start", time.ctime() #debug
    with open(json, 'r') as file:
        classifier = NaiveBayesClassifier(file, format='json')
        print "Classifier done!", time.ctime() #debug
    test = raw_input("Where is the test eml_folder?")
    print "Testing...", time.ctime()
    for emails in dir_list(test):
        print classifier.classify(emails)
    print "Testing done", time.ctime()

def run_test(train, test, name):
   print "Training..."
   cll = NaiveBayesClassifier(train)
   print "Done training\n"
   accuracy = cll.accuracy(test)
   print "Accuracy: " + str(accuracy)

   # get matching lists of predicted and true labels
   pred_labels = list()
   true_labels = list()
   for obj in test:
      prob_label = cll.prob_classify(obj[0]).max()
      true_label = obj[1]
      true_labels.append(true_label)
      pred_labels.append(prob_label)

   # transform our labels to numbers
   labels = cll.labels()
   i = 0
   label_num = dict()
   for label in labels:
      label_num[label] = i
      i = i + 1

   # match our predicted and true labels with the number representations
   true_label_nums = list()
   pred_label_nums = list()
   for true_l, pred_l in zip(true_labels, pred_labels):
      true_label_nums.append(label_num[true_l])
      pred_label_nums.append(label_num[pred_l])

   cm = confusion_matrix(true_label_nums, pred_label_nums)
   print cm
   print "\n"

   with open("test_results.txt", "a") as tr:
      tr.write(str(name) + "\n")
      tr.write(str(accuracy) + "\n")
      tr.write(str(cm))
      tr.write("\n\n")

   import matplotlib.pyplot as plt
   fig = plt.figure()
   ax = fig.add_subplot(111)
   cax = ax.matshow(cm)
   plt.title("Confusion Matrix For "+name)
   fig.colorbar(cax)
   ax.set_xticklabels(['']+labels)
   ax.set_yticklabels(['']+labels)
   plt.xlabel("Predicted")
   plt.ylabel("True")
   plt.savefig('plots/'+name+'.pdf', bbox_inches='tight') 

def main():
	print "This is Naive Bayes' Classifier..."

	#read training data
	#training_data = open("training_data").readlines()
	training_data = open("training_data_final").readlines()
	#load training data
	training_tuples = loadData(training_data)

	training_tuples_api = make_api_tuples(training_tuples)
	print training_tuples_api

	#display tuples
	#for t in training_tuples:
	#	t.show()

	#gather classes
	classes = filterClasses(training_tuples)
	#print "classes = ", classes

	#gather vocab
	vocab = getVocab(training_tuples)
	#print vocab

	#generate prior
	prior = generatePrior(training_tuples, classes)
	#print prior

	#generate likelihood
	likelihood = generateLikelihood(training_tuples, vocab, classes)
	#print likelihood

	#read test data
        #test_data = open("test_data").readlines()
        test_data = open("test_data_final").readlines()
        #load test data
        test_tuples = loadData(test_data)

	test_tuples_api = make_api_tuples(test_tuples)
	#calculate C-MAP
	posterior = predict(test_tuples, classes, prior, likelihood)
	showResults(training_data, test_data, posterior)

	#calculate accuracy
	evaluateAccuracy(test_tuples, posterior)

	#Naive Bayes API
	cl = NaiveBayesClassifier(training_tuples_api)
	# Compute accuracy
	print("Accuracy: {0}".format(cl.accuracy(test_tuples_api)))

def train(pos_examples, neg_examples, train_fraction=0.6):
    """Train a classifier, holding out train_fraction of pos_examples and neg_examples as a test set.
    Return the tuple:
        
        (the classifier, accuracy, positive test example list, negative test example list, )

    """

    pos_split = int(train_fraction * len(pos_examples))
    pos_train, pos_test = pos_examples[0:pos_split], pos_examples[pos_split:]
    neg_split = int(train_fraction * len(neg_examples))
    neg_train, neg_test = neg_examples[0:neg_split], neg_examples[neg_split:]

    cl = NaiveBayesClassifier(pos_train + neg_train)
    return cl, cl.accuracy(pos_test + neg_test), pos_test, neg_test

 def train(self, train_set):
     train_data = []
     for t in train_set:
         train_data.append((self._cvobj_to_string(t[0]),t[1]))
     print "Training model..."
     #print train_data
     self.cl = NaiveBayesClassifier(train_data)

class TimeLogicAdapter(LogicAdapter):
    """
    The TimeLogicAdapter returns the current time.
    """

    def __init__(self, **kwargs):
        super(TimeLogicAdapter, self).__init__(**kwargs)

        training_data = [
            ('what time is it', 1),
            ('do you know the time', 1),
            ('do you know what time it is', 1),
            ('what is the time', 1),
            ('it is time to go to sleep', 0),
            ('what is your favorite color', 0),
            ('i had a great time', 0),
            ('what is', 0)
        ]

        self.classifier = NaiveBayesClassifier(training_data)

    def process(self, statement):
        now = datetime.now()

        confidence = self.classifier.classify(statement.text.lower())
        response = Statement('The current time is ' + now.strftime('%I:%M %p'))

        return confidence, response