Python MemoryDescriptorIndex.iterdescriptors方法代码示例

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import iterdescriptors [as 别名]
    def test_remove(self):
        i = MemoryDescriptorIndex()
        descrs = [random_descriptor() for _ in xrange(100)]
        i.add_many_descriptors(descrs)
        ntools.assert_equal(len(i), 100)
        ntools.assert_equal(list(i.iterdescriptors()), descrs)

        # remove singles
        i.remove_descriptor(descrs[0].uuid())
        ntools.assert_equal(len(i), 99)
        ntools.assert_equal(set(i.iterdescriptors()),
                            set(descrs[1:]))

        # remove many
        rm_d = descrs[slice(45, 80, 3)]
        i.remove_many_descriptors((d.uuid() for d in rm_d))
        ntools.assert_equal(len(i), 99 - len(rm_d))
        ntools.assert_equal(set(i.iterdescriptors()),
                            set(descrs[1:]).difference(rm_d))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import iterdescriptors [as 别名]
 def test_iterdescrs(self):
     i = MemoryDescriptorIndex()
     descrs = [random_descriptor() for _ in xrange(100)]
     i.add_many_descriptors(descrs)
     ntools.assert_equal(set(i.iterdescriptors()),
                         set(descrs))

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import iterdescriptors [as 别名]

#.........这里部分代码省略.........

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        if len(self.ex_pos_descriptors) + \
                len(self.positive_descriptors) <= 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")
        # Not clearing index because this step is intended to be additive

        # build up new working index
        # TODO: Only query using new positives since previous queries
        for p in self.ex_pos_descriptors.itervalues():
            if p.uuid() not in self._wi_init_seeds:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    self.nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_init_seeds.add(p.uuid())
        for p in self.positive_descriptors:
            if p.uuid() not in self._wi_init_seeds:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    self.nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_init_seeds.add(p.uuid())

        # Make new relevancy index
        self._log.info("Creating new relevancy index over working index.")
        #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
        self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                   get_relevancy_index_impls)
        self.rel_index.build_index(self.working_index.iterdescriptors())

    def adjudicate(self, new_positives=(), new_negatives=(),
                   un_positives=(), un_negatives=()):
        """
        Update current state of working index positive and negative
        adjudications based on descriptor UUIDs.

        :param new_positives: Descriptors of elements in our working index to
            now be considered to be positively relevant.
        :type new_positives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param new_negatives: Descriptors of elements in our working index to
            now be considered to be negatively relevant.
        :type new_negatives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param un_positives: Descriptors of elements in our working index to now
            be considered not positive any more.
        :type un_positives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param un_negatives: Descriptors of elements in our working index to now
            be considered not negative any more.
        :type un_negatives: collections.Iterable[smqtk.representation.DescriptorElement]

        """
        with self.lock:
            self.positive_descriptors.update(new_positives)
            self.positive_descriptors.difference_update(un_positives)
            self.positive_descriptors.difference_update(new_negatives)

            self.negative_descriptors.update(new_negatives)
            self.negative_descriptors.difference_update(un_negatives)
            self.negative_descriptors.difference_update(new_positives)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import iterdescriptors [as 别名]

#.........这里部分代码省略.........
        pos_examples = (self.external_positive_descriptors |
                        self.positive_descriptors)
        if len(pos_examples) == 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")

        # Not clearing working index because this step is intended to be
        # additive.
        updated = False

        # adding to working index
        self._log.info("Building working index using %d positive examples "
                       "(%d external, %d adjudicated)",
                       len(pos_examples),
                       len(self.external_positive_descriptors),
                       len(self.positive_descriptors))
        # TODO: parallel_map and reduce with merge-dict
        for p in pos_examples:
            if p.uuid() not in self._wi_seeds_used:
                self._log.debug("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_seeds_used.add(p.uuid())
                updated = True

        # Make new relevancy index
        if updated:
            self._log.info("Creating new relevancy index over working index.")
            #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
            self.rel_index = plugin.from_plugin_config(
                self.rel_index_config, get_relevancy_index_impls()
            )
            self.rel_index.build_index(self.working_index.iterdescriptors())

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: No working index has been initialized.
            :meth:`update_working_index` should have been called after
            adjudicating some positive examples.
        :raises RuntimeError: There are no adjudications to run on. We must
            have at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # combine pos/neg adjudications + added external data descriptors
            pos = self.positive_descriptors | self.external_positive_descriptors
            neg = self.negative_descriptors | self.external_negative_descriptors

            if not pos:
                raise RuntimeError("Did not find at least one positive "
                                   "adjudication.")

            self._log.debug("Ranking working set with %d pos and %d neg total "
                            "examples.", len(pos), len(neg))
            element_probability_map = self.rel_index.rank(pos, neg)

            if self.results is None:
                self.results = IqrResultsDict()
            self.results.update(element_probability_map)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import iterdescriptors [as 别名]

#.........这里部分代码省略.........
            self.negative_descriptors.update(new_negatives)
            self.negative_descriptors.difference_update(un_negatives)
            self.negative_descriptors.difference_update(new_positives)

    def update_working_index(self, nn_index):
        """
        Initialize or update our current working index using the given
        :class:`.NearestNeighborsIndex` instance given our current positively
        labeled descriptor elements.

        We only query from the index for new positive elements since the last
        update or reset.

        :param nn_index: :class:`.NearestNeighborsIndex` to query from.
        :type nn_index: smqtk.algorithms.NearestNeighborsIndex

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        if len(self.positive_descriptors) <= 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")

        # Not clearing working index because this step is intended to be
        # additive.
        updated = False

        # adding to working index
        for p in self.positive_descriptors:
            if p.uuid() not in self._wi_seeds_used:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_seeds_used.add(p.uuid())
                updated = True

        # Make new relevancy index
        if updated:
            self._log.info("Creating new relevancy index over working index.")
            #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
            self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                       get_relevancy_index_impls())
            self.rel_index.build_index(self.working_index.iterdescriptors())

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: No working index has been initialized.
            :meth:`update_working_index` should have been called after
            adjudicating some positive examples.
        :raises RuntimeError: There are no adjudications to run on. We must
            have at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # fuse pos/neg adjudications + added positive data descriptors
            pos = self.positive_descriptors
            neg = self.negative_descriptors

            if not pos:
                raise RuntimeError("Did not find at least one positive "
                                   "adjudication.")

            element_probability_map = self.rel_index.rank(pos, neg)

            if self.results is None:
                self.results = IqrResultsDict()
            self.results.update(element_probability_map)

            # Force adjudicated positives and negatives to be probability 1 and
            # 0, respectively, since we want to control where they show up in
            # our results view.
            # - Not all pos/neg descriptors may be in our working index.
            for d in pos:
                if d in self.results:
                    self.results[d] = 1.0
            for d in neg:
                if d in self.results:
                    self.results[d] = 0.0

    def reset(self):
        """ Reset the IQR Search state

        No positive adjudications, reload original feature data

        """
        with self.lock:
            self.working_index.clear()
            self._wi_seeds_used.clear()
            self.positive_descriptors.clear()
            self.negative_descriptors.clear()

            self.rel_index = None
            self.results = None