Python Corpus.save_to_file方法代码示例

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import save_to_file [as 别名]
def process_corpus(tr_in_filename, te_in_filename, u_in_filename,
                   tr_out_filename, te_out_filename, u_out_filename):
    input_f = open(tr_in_filename, 'r')
    tr_original_corpus = pickle.load(input_f)
    input_f.close()

    input_f = open(te_in_filename, 'r')
    te_original_corpus = pickle.load(input_f)
    input_f.close()

    input_f = open(u_in_filename, 'r')
    u_original_corpus = pickle.load(input_f)
    input_f.close()
    tr_instances = [d['question'] for d in tr_original_corpus
                    if '' not in d['target']]
    te_instances = [d['question'] for d in te_original_corpus
                    if '' not in d['target']]
    u_instances = [d['question'] for d in u_original_corpus
                   if ((not 'target' in d) or '' not in d['target'])]

    vect = get_features()
    vect.fit(tr_instances + te_instances + u_instances)
    v_instances = vect.transform(tr_instances + te_instances + u_instances)
    v_instances = csr_matrix(v_instances > 0, dtype=int8)
    print v_instances.shape

    tr_corpus = Corpus()
    tr_corpus.instances = v_instances[:len(tr_instances)]
    tr_corpus.full_targets = [d['target'] for d in tr_original_corpus
                              if '' not in d['target']]
    tr_corpus.representations = [_get_repr(i[0]) for i in tr_instances]
    tr_corpus._features_vectorizer = vect
    tr_corpus.save_to_file(tr_out_filename)

    te_corpus = Corpus()
    te_corpus.instances = v_instances[:len(te_instances)]
    te_corpus.full_targets = [d['target'] for d in te_original_corpus
                              if '' not in d['target']]
    te_corpus.representations = [_get_repr(i[0]) for i in te_instances]
    te_corpus._features_vectorizer = vect
    te_corpus.save_to_file(te_out_filename)

    u_corpus = Corpus()
    u_corpus.instances = v_instances[:len(u_instances)]
    u_corpus.full_targets = [d['target']
                             if ('target' in d and '' not in d['target']) else []
                             for d in u_original_corpus]
    u_corpus.representations = [_get_repr(i[0]) for i in u_instances]
    u_corpus._features_vectorizer = vect
    u_corpus.save_to_file(u_out_filename)

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import save_to_file [as 别名]

#.........这里部分代码省略.........
            The score of the classifier over the test corpus
        """
        return self.classifier.score(self.test_corpus.instances,
                                     self.test_corpus.primary_targets)

    def evaluate_training(self):
        """Evaluate the accuracy of the classifier with the labeled data.

        Returns:
            The score of the classifier over the training corpus
        """
        # Agregamos la evidencia del usuario para evaluacion?
        return self.classifier.score(self.training_corpus.instances,
                                     self.training_corpus.primary_targets)

    def get_report(self):
        """
        Returns:
            A sklearn.metrics.classification_report on the performance
            of the cassifier over the test corpus.
        """
        predicted_targets = self.predict(self.test_corpus.instances)
        return classification_report(self.test_corpus.primary_targets,
                                     predicted_targets)

    def label_corpus(self):
        """Adds the user corpus to the unlabeled_corpus and saves it in a file.

        The filename must be passed into the configuration under the name
        u_corpus_f.
        """
        if len(self.user_corpus):
            self.unlabeled_corpus.concetenate_corpus(self.user_corpus)
        self.unlabeled_corpus.save_to_file(self.u_corpus_f)

    def label_feature_corpus(self):
        """Adds user_features and asked_features in feature_corpus and saves it.

        The filename must be passed into the configuration under the name
        feature_corpus_f.
        """
        self.feature_corpus = np.where(self.asked_features,
                                       np.zeros((self.n_class, self.n_feat)),
                                       self.feature_corpus)

        self.feature_corpus = np.where(
            self.user_features > self.alpha,
            np.ones((self.n_class, self.n_feat)),
            self.feature_corpus
        )
        f = open(self.feature_corpus_f, 'w')
        pickle.dump(self.feature_corpus, f)
        f.close()

    def save_session(self, filename):
        """Saves the instances and targets introduced by the user in filename.

        Writes a pickle tuple in the file that can be recovered using the
        method load_session.

        Returns:
            False in case of error, True in case of success.
        """
        if not filename:
            return False
        if not (len(self.user_corpus) != None or self.user_features != None):

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import save_to_file [as 别名]
class TestCorpus(unittest.TestCase):

    def setUp(self):
        self.co = Corpus()
        self.co.instances = csr_matrix([[1, 2, 3], [4, 5, 6]])
        self.co.full_targets = [[1], [2,3]]
        self.co.representations = ['representation1', 'representation2']
        self.co.calculate_primary_targets()
        self.co.add_extra_info('extra_info1')

    def tearDown(self):
        self.assertTrue(self.co.check_consistency())

    def test_load_and_save(self):
        """Load and save functions must be inverses."""
        filename = 'testing_file'
        self.co.save_to_file(filename)
        new_co = Corpus()
        new_co.load_from_file(filename)
        self.assertTrue(_eq_crs_matrix(new_co.instances, self.co.instances))
        for index in range(len(self.co)):
            self.assertEqual(self.co.full_targets[index],
                             new_co.full_targets[index])
            self.assertEqual(self.co.representations[index],
                             new_co.representations[index])
        self.assertIsNotNone(new_co.primary_targets)

    def test_add_instance(self):
        self.co.add_instance([2, 3, 4], [2], 'representation3')
        self.assertEqual(len(self.co), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([2, 3, 4]),
                                       self.co.instances[-1]))

    def test_add_extra_info(self):
        self.assertEqual(len(self.co.extra_info), 1)
        self.assertIn('extra_info1', self.co.extra_info)

    def test_add_extra_info_twice(self):
        self.co.add_extra_info('extra_info1', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 1)
        self.assertIn('extra_info1', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info1'], [1, 1])

    def test_add_extra_info_second(self):
        self.co.add_extra_info('extra_info2', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 2)
        self.assertIn('extra_info2', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info2'], [1, 1])

    def test_pop_first_instance(self):
        result = self.co.pop_instance(0)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]), result[0]))
        self.assertEqual(result[1], [1])
        self.assertEqual(result[2], 'representation1')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [2, 3])
        self.assertEqual(self.co.representations[0], 'representation2')

    def test_pop_last_instance(self):
        result = self.co.pop_instance(1)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [1])
        self.assertEqual(self.co.representations[0], 'representation1')

    def test_pop_middle_instace(self):
        self.co.add_instance([2, 3, 4], [2], 'representation3')
        result = self.co.pop_instance(1)

        self.assertEqual(len(self.co), 2)
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')

    def test_pop_last_instance(self):
        self.co.pop_instance(1)
        self.co.pop_instance(0)

        self.assertEqual(len(self.co), 0)

    def test_concatenate_corpus(self):
        new_corpus = Corpus()
        new_corpus.add_instance([2, 3, 4], [2], 'representation3')
        new_corpus.add_instance([10, 4, 4], [1, 1, 2], 'representation3')
        new_corpus.calculate_primary_targets()
        self.assertTrue(new_corpus.add_extra_info('extra_info1'))

#.........这里部分代码省略.........

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import save_to_file [as 别名]
def count_by_class(corpus):
    """Returns a dictionary with the number of instances by class"""
    result = defaultdict(lambda: 0)
    for target in co.primary_targets:
        result[target] += 1
    return result

c_by_class = count_by_class(co)

for k, v in c_by_class.items():
    print k, v

limit = sorted(c_by_class.values())[-2]
# limit = 10
to_remove = c_by_class['other'] - limit
to_remove = {k: c_by_class[k] - limit for k in c_by_class}
print to_remove, limit

for i in range(len(co)-1, 0, -1):
    target = co.primary_targets[i]
    if to_remove[target] > 0:
        co.pop_instance(i)
        to_remove[target] -= 1

c_by_class = count_by_class(co)

for k, v in c_by_class.items():
    print k, v

co.save_to_file('experimental/unlabeled_new_corpus_balanced2.pickle')