Python MemoryDescriptorIndex.add_many_descriptors方法代码示例

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_has(self):
        i = MemoryDescriptorIndex()
        descrs = [random_descriptor() for _ in xrange(10)]
        i.add_many_descriptors(descrs)

        ntools.assert_true(i.has_descriptor(descrs[4].uuid()))
        ntools.assert_false(i.has_descriptor('not_an_int'))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_added_descriptor_table_caching(self):
        cache_elem = DataMemoryElement(readonly=False)
        descrs = [random_descriptor() for _ in range(3)]
        expected_table = dict((r.uuid(), r) for r in descrs)

        i = MemoryDescriptorIndex(cache_elem)
        self.assertTrue(cache_elem.is_empty())

        # Should add descriptors to table, caching to writable element.
        i.add_many_descriptors(descrs)
        self.assertFalse(cache_elem.is_empty())
        self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
                         expected_table)

        # Changing the internal table (remove, add) it should reflect in
        # cache
        new_d = random_descriptor()
        expected_table[new_d.uuid()] = new_d
        i.add_descriptor(new_d)
        self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
                         expected_table)

        rm_d = list(expected_table.values())[0]
        del expected_table[rm_d.uuid()]
        i.remove_descriptor(rm_d.uuid())
        self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
                         expected_table)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_clear(self):
        i = MemoryDescriptorIndex()
        n = 10

        descrs = [random_descriptor() for _ in xrange(n)]
        i.add_many_descriptors(descrs)
        ntools.assert_equal(len(i), n)
        i.clear()
        ntools.assert_equal(len(i), 0)
        ntools.assert_equal(i._table, {})

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_update_index_existing_descriptors_frozenset(self):
        """
        Same as ``test_update_index_similar_descriptors`` but testing that
        we can update the index when seeded with structures with existing
        values.
        """
        # Similar Descriptors to build and update on (different instances)
        descriptors1 = [
            DescriptorMemoryElement('t', 0).set_vector([0]),
            DescriptorMemoryElement('t', 1).set_vector([1]),
            DescriptorMemoryElement('t', 2).set_vector([2]),
            DescriptorMemoryElement('t', 3).set_vector([3]),
            DescriptorMemoryElement('t', 4).set_vector([4]),
        ]
        descriptors2 = [
            DescriptorMemoryElement('t', 5).set_vector([0]),
            DescriptorMemoryElement('t', 6).set_vector([1]),
            DescriptorMemoryElement('t', 7).set_vector([2]),
            DescriptorMemoryElement('t', 8).set_vector([3]),
            DescriptorMemoryElement('t', 9).set_vector([4]),
        ]

        descr_index = MemoryDescriptorIndex()
        descr_index.add_many_descriptors(descriptors1)

        hash_kvs = MemoryKeyValueStore()
        hash_kvs.add(0, frozenset({0}))
        hash_kvs.add(1, frozenset({1}))
        hash_kvs.add(2, frozenset({2}))
        hash_kvs.add(3, frozenset({3}))
        hash_kvs.add(4, frozenset({4}))

        index = LSHNearestNeighborIndex(DummyHashFunctor(),
                                        descr_index, hash_kvs)
        index.update_index(descriptors2)

        assert descr_index.count() == 10
        # Above descriptors should be considered "in" the descriptor set now.
        for d in descriptors1:
            assert d in descr_index
        for d in descriptors2:
            assert d in descr_index
        # Known hashes of the above descriptors should be in the KVS
        assert set(hash_kvs.keys()) == {0, 1, 2, 3, 4}
        assert hash_kvs.get(0) == {0, 5}
        assert hash_kvs.get(1) == {1, 6}
        assert hash_kvs.get(2) == {2, 7}
        assert hash_kvs.get(3) == {3, 8}
        assert hash_kvs.get(4) == {4, 9}

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_count(self):
        index = MemoryDescriptorIndex()
        self.assertEqual(index.count(), 0)

        d1 = random_descriptor()
        index.add_descriptor(d1)
        self.assertEqual(index.count(), 1)

        d2, d3, d4 = (random_descriptor(),
                      random_descriptor(),
                      random_descriptor())
        index.add_many_descriptors([d2, d3, d4])
        self.assertEqual(index.count(), 4)

        d5 = random_descriptor()
        index.add_descriptor(d5)
        self.assertEqual(index.count(), 5)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_remove(self):
        i = MemoryDescriptorIndex()
        descrs = [random_descriptor() for _ in xrange(100)]
        i.add_many_descriptors(descrs)
        ntools.assert_equal(len(i), 100)
        ntools.assert_equal(list(i.iterdescriptors()), descrs)

        # remove singles
        i.remove_descriptor(descrs[0].uuid())
        ntools.assert_equal(len(i), 99)
        ntools.assert_equal(set(i.iterdescriptors()),
                            set(descrs[1:]))

        # remove many
        rm_d = descrs[slice(45, 80, 3)]
        i.remove_many_descriptors((d.uuid() for d in rm_d))
        ntools.assert_equal(len(i), 99 - len(rm_d))
        ntools.assert_equal(set(i.iterdescriptors()),
                            set(descrs[1:]).difference(rm_d))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_get_descriptors(self):
        descrs = [
            random_descriptor(),   # [0]
            random_descriptor(),   # [1]
            random_descriptor(),   # [2]
            random_descriptor(),   # [3]
            random_descriptor(),   # [4]
        ]
        index = MemoryDescriptorIndex()
        index.add_many_descriptors(descrs)

        # single descriptor reference
        r = index.get_descriptor(descrs[1].uuid())
        ntools.assert_equal(r, descrs[1])

        # multiple descriptor reference
        r = list(index.get_many_descriptors([descrs[0].uuid(),
                                             descrs[3].uuid()]))
        ntools.assert_equal(len(r), 2)
        ntools.assert_equal(set(r),
                            {descrs[0], descrs[3]})

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_table_caching(self):
        fd, tmp_cache = tempfile.mkstemp()
        os.close(fd)
        os.remove(tmp_cache)

        try:
            i = MemoryDescriptorIndex(tmp_cache)
            descrs = [random_descriptor() for _ in xrange(3)]
            expected_cache = dict((r.uuid(), r) for r in descrs)

            # cache should not exist yet
            ntools.assert_false(os.path.isfile(tmp_cache))

            # Should write file and should be a dictionary of 3
            # elements
            i.add_many_descriptors(descrs)
            ntools.assert_true(os.path.isfile(tmp_cache))
            with open(tmp_cache) as f:
                ntools.assert_equal(cPickle.load(f),
                                    expected_cache)

            # Changing the internal table (remove, add) it should reflect in
            # cache
            new_d = random_descriptor()
            i.add_descriptor(new_d)
            expected_cache[new_d.uuid()] = new_d
            with open(tmp_cache) as f:
                ntools.assert_equal(cPickle.load(f),
                                    expected_cache)

            rm_d = expected_cache.values()[0]
            i.remove_descriptor(rm_d.uuid())
            del expected_cache[rm_d.uuid()]
            with open(tmp_cache) as f:
                ntools.assert_equal(cPickle.load(f),
                                    expected_cache)
        finally:
            os.remove(tmp_cache)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
    def test_add_many(self):
        descrs = [
            random_descriptor(),
            random_descriptor(),
            random_descriptor(),
            random_descriptor(),
            random_descriptor(),
        ]
        index = MemoryDescriptorIndex()
        index.add_many_descriptors(descrs)

        # Compare code keys of input to code keys in internal table
        ntools.assert_equal(set(index._table.keys()),
                            set([e.uuid() for e in descrs]))

        # Get the set of descriptors in the internal table and compare it with
        # the set of generated random descriptors.
        r_set = set()
        [r_set.add(d) for d in index._table.values()]
        ntools.assert_equal(
            set([e for e in descrs]),
            r_set
        )

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
 def test_iteritems(self):
     i = MemoryDescriptorIndex()
     descrs = [random_descriptor() for _ in xrange(100)]
     i.add_many_descriptors(descrs)
     ntools.assert_equal(set(i.iteritems()),
                         set((d.uuid(), d) for d in descrs))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]

#.........这里部分代码省略.........
        :class:`.NearestNeighborsIndex` instance given our current positively
        labeled descriptor elements.

        We only query from the index for new positive elements since the last
        update or reset.

        :param nn_index: :class:`.NearestNeighborsIndex` to query from.
        :type nn_index: smqtk.algorithms.NearestNeighborsIndex

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        pos_examples = (self.external_positive_descriptors |
                        self.positive_descriptors)
        if len(pos_examples) == 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")

        # Not clearing working index because this step is intended to be
        # additive.
        updated = False

        # adding to working index
        self._log.info("Building working index using %d positive examples "
                       "(%d external, %d adjudicated)",
                       len(pos_examples),
                       len(self.external_positive_descriptors),
                       len(self.positive_descriptors))
        # TODO: parallel_map and reduce with merge-dict
        for p in pos_examples:
            if p.uuid() not in self._wi_seeds_used:
                self._log.debug("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_seeds_used.add(p.uuid())
                updated = True

        # Make new relevancy index
        if updated:
            self._log.info("Creating new relevancy index over working index.")
            #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
            self.rel_index = plugin.from_plugin_config(
                self.rel_index_config, get_relevancy_index_impls()
            )
            self.rel_index.build_index(self.working_index.iterdescriptors())

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: No working index has been initialized.
            :meth:`update_working_index` should have been called after
            adjudicating some positive examples.
        :raises RuntimeError: There are no adjudications to run on. We must
            have at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # combine pos/neg adjudications + added external data descriptors
            pos = self.positive_descriptors | self.external_positive_descriptors
            neg = self.negative_descriptors | self.external_negative_descriptors

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]

#.........这里部分代码省略.........

        """
        with self.lock:
            r = self.descriptor.compute_descriptor_async(
                data_elements, self.descriptor_factory
            )
            for da in r:
                self.ex_neg_descriptors[da.uuid()] = r[da]
                self.ex_data[da.uuid()] = da

    def initialize(self):
        """
        Initialize working index based on currently set positive exemplar data.

        This takes into account the currently set positive data descriptors as
        well as positively adjudicated descriptors from the lifetime of this
        session.

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        if len(self.ex_pos_descriptors) + \
                len(self.positive_descriptors) <= 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")
        # Not clearing index because this step is intended to be additive

        # build up new working index
        # TODO: Only query using new positives since previous queries
        for p in self.ex_pos_descriptors.itervalues():
            if p.uuid() not in self._wi_init_seeds:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    self.nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_init_seeds.add(p.uuid())
        for p in self.positive_descriptors:
            if p.uuid() not in self._wi_init_seeds:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    self.nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_init_seeds.add(p.uuid())

        # Make new relevancy index
        self._log.info("Creating new relevancy index over working index.")
        #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
        self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                   get_relevancy_index_impls)
        self.rel_index.build_index(self.working_index.iterdescriptors())

    def adjudicate(self, new_positives=(), new_negatives=(),
                   un_positives=(), un_negatives=()):
        """
        Update current state of working index positive and negative
        adjudications based on descriptor UUIDs.

        :param new_positives: Descriptors of elements in our working index to
            now be considered to be positively relevant.
        :type new_positives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param new_negatives: Descriptors of elements in our working index to
            now be considered to be negatively relevant.
        :type new_negatives: collections.Iterable[smqtk.representation.DescriptorElement]

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]

#.........这里部分代码省略.........

            self.negative_descriptors.update(new_negatives)
            self.negative_descriptors.difference_update(un_negatives)
            self.negative_descriptors.difference_update(new_positives)

    def update_working_index(self, nn_index):
        """
        Initialize or update our current working index using the given
        :class:`.NearestNeighborsIndex` instance given our current positively
        labeled descriptor elements.

        We only query from the index for new positive elements since the last
        update or reset.

        :param nn_index: :class:`.NearestNeighborsIndex` to query from.
        :type nn_index: smqtk.algorithms.NearestNeighborsIndex

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        if len(self.positive_descriptors) <= 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")

        # Not clearing working index because this step is intended to be
        # additive.
        updated = False

        # adding to working index
        for p in self.positive_descriptors:
            if p.uuid() not in self._wi_seeds_used:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_seeds_used.add(p.uuid())
                updated = True

        # Make new relevancy index
        if updated:
            self._log.info("Creating new relevancy index over working index.")
            #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
            self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                       get_relevancy_index_impls())
            self.rel_index.build_index(self.working_index.iterdescriptors())

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: No working index has been initialized.
            :meth:`update_working_index` should have been called after
            adjudicating some positive examples.
        :raises RuntimeError: There are no adjudications to run on. We must
            have at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # fuse pos/neg adjudications + added positive data descriptors
            pos = self.positive_descriptors
            neg = self.negative_descriptors

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
 def test_iteritems(self):
     i = MemoryDescriptorIndex()
     descrs = [random_descriptor() for _ in range(100)]
     i.add_many_descriptors(descrs)
     self.assertEqual(set(six.iteritems(i)),
                      set((d.uuid(), d) for d in descrs))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import add_many_descriptors [as 别名]
 def test_iterdescrs(self):
     i = MemoryDescriptorIndex()
     descrs = [random_descriptor() for _ in range(100)]
     i.add_many_descriptors(descrs)
     self.assertEqual(set(i.iterdescriptors()),
                      set(descrs))