Python Corpus.pop_instance方法代码示例

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import pop_instance [as 别名]
class TestCorpus(unittest.TestCase):

    def setUp(self):
        self.co = Corpus()
        self.co.instances = csr_matrix([[1, 2, 3], [4, 5, 6]])
        self.co.full_targets = [[1], [2,3]]
        self.co.representations = ['representation1', 'representation2']
        self.co.calculate_primary_targets()
        self.co.add_extra_info('extra_info1')

    def tearDown(self):
        self.assertTrue(self.co.check_consistency())

    def test_load_and_save(self):
        """Load and save functions must be inverses."""
        filename = 'testing_file'
        self.co.save_to_file(filename)
        new_co = Corpus()
        new_co.load_from_file(filename)
        self.assertTrue(_eq_crs_matrix(new_co.instances, self.co.instances))
        for index in range(len(self.co)):
            self.assertEqual(self.co.full_targets[index],
                             new_co.full_targets[index])
            self.assertEqual(self.co.representations[index],
                             new_co.representations[index])
        self.assertIsNotNone(new_co.primary_targets)

    def test_add_instance(self):
        self.co.add_instance([2, 3, 4], [2], 'representation3')
        self.assertEqual(len(self.co), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([2, 3, 4]),
                                       self.co.instances[-1]))

    def test_add_extra_info(self):
        self.assertEqual(len(self.co.extra_info), 1)
        self.assertIn('extra_info1', self.co.extra_info)

    def test_add_extra_info_twice(self):
        self.co.add_extra_info('extra_info1', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 1)
        self.assertIn('extra_info1', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info1'], [1, 1])

    def test_add_extra_info_second(self):
        self.co.add_extra_info('extra_info2', values=[1, 1])
        self.assertEqual(len(self.co.extra_info), 2)
        self.assertIn('extra_info2', self.co.extra_info)
        self.assertEqual(self.co.extra_info['extra_info2'], [1, 1])

    def test_pop_first_instance(self):
        result = self.co.pop_instance(0)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]), result[0]))
        self.assertEqual(result[1], [1])
        self.assertEqual(result[2], 'representation1')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [2, 3])
        self.assertEqual(self.co.representations[0], 'representation2')

    def test_pop_last_instance(self):
        result = self.co.pop_instance(1)
        # check result
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')
        #check corpus
        self.assertEqual(len(self.co), 1)
        self.assertTrue(_eq_crs_matrix(csr_matrix([1, 2, 3]),
                                       self.co.instances[0]))
        self.assertEqual(self.co.full_targets[0], [1])
        self.assertEqual(self.co.representations[0], 'representation1')

    def test_pop_middle_instace(self):
        self.co.add_instance([2, 3, 4], [2], 'representation3')
        result = self.co.pop_instance(1)

        self.assertEqual(len(self.co), 2)
        self.assertEqual(len(result), 3)
        self.assertTrue(_eq_crs_matrix(csr_matrix([4, 5, 6]), result[0]))
        self.assertEqual(result[1], [2, 3])
        self.assertEqual(result[2], 'representation2')

    def test_pop_last_instance(self):
        self.co.pop_instance(1)
        self.co.pop_instance(0)

        self.assertEqual(len(self.co), 0)

    def test_concatenate_corpus(self):
        new_corpus = Corpus()
        new_corpus.add_instance([2, 3, 4], [2], 'representation3')
        new_corpus.add_instance([10, 4, 4], [1, 1, 2], 'representation3')
        new_corpus.calculate_primary_targets()
        self.assertTrue(new_corpus.add_extra_info('extra_info1'))

#.........这里部分代码省略.........

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import pop_instance [as 别名]

#.........这里部分代码省略.........
        result = 0
        while ((not max_iterations or it < max_iterations) and
              len(self.unlabeled_corpus)):
            it += 1
            new_index = self.get_next_instance()
            try:
                new_instance = self.unlabeled_corpus.instances[new_index]
            except IndexError:
                import ipdb; ipdb.set_trace()
            representation = self.unlabeled_corpus.representations[new_index]
            if (self.emulate and
                self.unlabeled_corpus.primary_targets[new_index]):
                prediction = self.unlabeled_corpus.primary_targets[new_index]
                message = "Emulation: Adding instance {}, {}".format(
                    representation, prediction
                )
                print colored(message, "red")
            if (not self.emulate or
                not self.unlabeled_corpus.primary_targets[new_index]):
                classes = self._most_probable_classes(new_instance)
                prediction = get_labeled_instance(representation, classes)
            if prediction == 'stop':
                break
            if prediction == 'train':
                self._train()
                self._expectation_maximization()
                continue
            # if prediction == 'other':
            #     self.unlabeled_corpus.pop_instance(new_index)
            #     continue

            self.new_instances += 1
            result += 1
            instance, targets, r = self.unlabeled_corpus.pop_instance(new_index)
            self.user_corpus.add_instance(
                instance, [prediction] + targets, r
            )

        return result

    def feature_bootstrap(self, get_class, get_labeled_features,
                          max_iterations=None):
        """Presents a class and possible features until the prediction is stop.

        Args:
            get_class: A function that receives a list of classes and returns
            one of them. Can return None in case of error.
            get_labeled_features: A function that receives a class and a list
            of features. It must return a list of features associated with the
            class. Can return None in case of error.
            max_iterations: Optional. An interger. The cicle will execute at
            most max_iterations times if the user does not enter stop before.

        Returns:
            The number of features the user has labeled.
        """
        result = 0
        while not max_iterations or result < max_iterations:
            class_name = get_class(self.get_class_options())
            if not class_name:
                continue
            if class_name == 'stop':
                break
            if class_name == 'train':
                self._train()
                self._expectation_maximization()

# 需要导入模块: from corpus import Corpus [as 别名]
# 或者: from corpus.Corpus import pop_instance [as 别名]
def count_by_class(corpus):
    """Returns a dictionary with the number of instances by class"""
    result = defaultdict(lambda: 0)
    for target in co.primary_targets:
        result[target] += 1
    return result

c_by_class = count_by_class(co)

for k, v in c_by_class.items():
    print k, v

limit = sorted(c_by_class.values())[-2]
# limit = 10
to_remove = c_by_class['other'] - limit
to_remove = {k: c_by_class[k] - limit for k in c_by_class}
print to_remove, limit

for i in range(len(co)-1, 0, -1):
    target = co.primary_targets[i]
    if to_remove[target] > 0:
        co.pop_instance(i)
        to_remove[target] -= 1

c_by_class = count_by_class(co)

for k, v in c_by_class.items():
    print k, v

co.save_to_file('experimental/unlabeled_new_corpus_balanced2.pickle')