Python Corpus.concetenate_corpus方法代码示例

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import concetenate_corpus [as 别名]
from corpus import Corpus
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix, classification_report

co = Corpus()
c2 = Corpus()
ct = Corpus()

co.load_from_file('corpus/experimental2/unlabeled_new_corpus.pickle')
ct.load_from_file('corpus/experimental2/test_new_corpus.pickle')
c2.load_from_file('corpus/experimental2/training_new_corpus.pickle')

co.concetenate_corpus(c2)

mnb = MultinomialNB()
mnb.fit(co.instances, co.primary_targets)
print mnb.score(ct.instances, ct.primary_targets)
predicted_targets = mnb.predict(ct.instances)
print classification_report(ct.primary_targets, predicted_targets)
cm = confusion_matrix(ct.primary_targets, predicted_targets)
for index, row in enumerate(cm):
    print mnb.classes_[index], [(mnb.classes_[j], row[j])
                                for j in range(len(row))
                                if row[j]]

new_q = 0
for index, row in enumerate(cm):
    if mnb.classes_[index] != 'other':
        new_q += cm[index][index]
print new_q / float(cm.sum()-135)

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import concetenate_corpus [as 别名]

#.........这里部分代码省略.........
        Returns:
            The score of the classifier over the test corpus
        """
        return self.classifier.score(self.test_corpus.instances,
                                     self.test_corpus.primary_targets)

    def evaluate_training(self):
        """Evaluate the accuracy of the classifier with the labeled data.

        Returns:
            The score of the classifier over the training corpus
        """
        # Agregamos la evidencia del usuario para evaluacion?
        return self.classifier.score(self.training_corpus.instances,
                                     self.training_corpus.primary_targets)

    def get_report(self):
        """
        Returns:
            A sklearn.metrics.classification_report on the performance
            of the cassifier over the test corpus.
        """
        predicted_targets = self.predict(self.test_corpus.instances)
        return classification_report(self.test_corpus.primary_targets,
                                     predicted_targets)

    def label_corpus(self):
        """Adds the user corpus to the unlabeled_corpus and saves it in a file.

        The filename must be passed into the configuration under the name
        u_corpus_f.
        """
        if len(self.user_corpus):
            self.unlabeled_corpus.concetenate_corpus(self.user_corpus)
        self.unlabeled_corpus.save_to_file(self.u_corpus_f)

    def label_feature_corpus(self):
        """Adds user_features and asked_features in feature_corpus and saves it.

        The filename must be passed into the configuration under the name
        feature_corpus_f.
        """
        self.feature_corpus = np.where(self.asked_features,
                                       np.zeros((self.n_class, self.n_feat)),
                                       self.feature_corpus)

        self.feature_corpus = np.where(
            self.user_features > self.alpha,
            np.ones((self.n_class, self.n_feat)),
            self.feature_corpus
        )
        f = open(self.feature_corpus_f, 'w')
        pickle.dump(self.feature_corpus, f)
        f.close()

    def save_session(self, filename):
        """Saves the instances and targets introduced by the user in filename.

        Writes a pickle tuple in the file that can be recovered using the
        method load_session.

        Returns:
            False in case of error, True in case of success.
        """
        if not filename:
            return False

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import concetenate_corpus [as 别名]

#.........这里部分代码省略.........
    def test_add_extra_info_twice(self):
        self.co.add_extra_info('extra_info1', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 1)
        self.assertIn('extra_info1', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info1'], [1, 1])

    def test_add_extra_info_second(self):
        self.co.add_extra_info('extra_info2', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 2)
        self.assertIn('extra_info2', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info2'], [1, 1])

    def test_pop_first_instance(self):
        result = self.co.pop_instance(0)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]), result[0]))
        self.assertEqual(result[1], [1])
        self.assertEqual(result[2], 'representation1')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [2, 3])
        self.assertEqual(self.co.representations[0], 'representation2')

    def test_pop_last_instance(self):
        result = self.co.pop_instance(1)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [1])
        self.assertEqual(self.co.representations[0], 'representation1')

    def test_pop_middle_instace(self):
        self.co.add_instance([2, 3, 4], [2], 'representation3')
        result = self.co.pop_instance(1)

        self.assertEqual(len(self.co), 2)
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')

    def test_pop_last_instance(self):
        self.co.pop_instance(1)
        self.co.pop_instance(0)

        self.assertEqual(len(self.co), 0)

    def test_concatenate_corpus(self):
        new_corpus = Corpus()
        new_corpus.add_instance([2, 3, 4], [2], 'representation3')
        new_corpus.add_instance([10, 4, 4], [1, 1, 2], 'representation3')
        new_corpus.calculate_primary_targets()
        self.assertTrue(new_corpus.add_extra_info('extra_info1'))

        self.co.concetenate_corpus(new_corpus)
        self.assertEqual(len(self.co), 4)

    def test_concatenate_corpus_no_extra_info(self):
        new_corpus = Corpus()
        new_corpus.add_instance([2, 3, 4], [2], 'representation3')
        new_corpus.add_instance([10, 4, 4], [1, 1, 2], 'representation3')
        new_corpus.calculate_primary_targets()

        self.co.concetenate_corpus(new_corpus)
        self.assertEqual(len(self.co), 4)

    def test_concatenate_empty_corpus(self):
        new_corpus = Corpus()
        self.assertTrue(new_corpus.add_extra_info('extra_info1'))
        self.co.concetenate_corpus(new_corpus)
        self.assertEqual(len(self.co), 2)

    def test_calculate_primary_targets(self):
        self.assertEqual(self.co.primary_targets, [1,2])

    def test_primary_targets_none(self):
        self.co.add_instance([0, 2, 3], [], 'r')
        self.co.calculate_primary_targets()
        self.assertEqual(self.co.primary_targets, [1, 2, None])

    def test_primary_targets_mode(self):
        self.co.add_instance([0, 2, 3], [1, 4, 4, 3, 3], 'r')
        self.co.calculate_primary_targets()
        self.assertEqual(self.co.primary_targets, [1, 2, 3])

    def test_len(self):
        self.assertEqual(len(self.co), 2)

    def test_len_empty(self):
        new_corpus = Corpus()
        self.assertEqual(len(new_corpus), 0)