Python MemoryDescriptorIndex.clear方法代码示例

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import clear [as 别名]
    def test_clear(self):
        i = MemoryDescriptorIndex()
        n = 10

        descrs = [random_descriptor() for _ in xrange(n)]
        i.add_many_descriptors(descrs)
        ntools.assert_equal(len(i), n)
        i.clear()
        ntools.assert_equal(len(i), 0)
        ntools.assert_equal(i._table, {})

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import clear [as 别名]

#.........这里部分代码省略.........
                    self.nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_init_seeds.add(p.uuid())

        # Make new relevancy index
        self._log.info("Creating new relevancy index over working index.")
        #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
        self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                   get_relevancy_index_impls)
        self.rel_index.build_index(self.working_index.iterdescriptors())

    def adjudicate(self, new_positives=(), new_negatives=(),
                   un_positives=(), un_negatives=()):
        """
        Update current state of working index positive and negative
        adjudications based on descriptor UUIDs.

        :param new_positives: Descriptors of elements in our working index to
            now be considered to be positively relevant.
        :type new_positives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param new_negatives: Descriptors of elements in our working index to
            now be considered to be negatively relevant.
        :type new_negatives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param un_positives: Descriptors of elements in our working index to now
            be considered not positive any more.
        :type un_positives: collections.Iterable[smqtk.representation.DescriptorElement]

        :param un_negatives: Descriptors of elements in our working index to now
            be considered not negative any more.
        :type un_negatives: collections.Iterable[smqtk.representation.DescriptorElement]

        """
        with self.lock:
            self.positive_descriptors.update(new_positives)
            self.positive_descriptors.difference_update(un_positives)
            self.positive_descriptors.difference_update(new_negatives)

            self.negative_descriptors.update(new_negatives)
            self.negative_descriptors.difference_update(un_negatives)
            self.negative_descriptors.difference_update(new_positives)

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: There are no adjudications to run on. We must have
            at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # fuse pos/neg adjudications + added positive data descriptors
            pos = self.ex_pos_descriptors.values() + list(self.positive_descriptors)
            neg = self.ex_neg_descriptors.values() + list(self.negative_descriptors)

            if not pos:
                raise RuntimeError("Did not find at least one positive "
                                   "adjudication.")

            id_probability_map = self.rel_index.rank(pos, neg)

            if self.results is None:
                self.results = IqrResultsDict()
            self.results.update(id_probability_map)

            # Force adjudicated positives and negatives to be probability 1 and
            # 0, respectively, since we want to control where they show up in
            # our results view.
            # - Not all pos/neg descriptors may be in our working index.
            for d in pos:
                if d in self.results:
                    self.results[d] = 1.0
            for d in neg:
                if d in self.results:
                    self.results[d] = 0.0

    def reset(self):
        """ Reset the IQR Search state

        No positive adjudications, reload original feature data

        """
        with self.lock:
            self.working_index.clear()
            self._wi_init_seeds.clear()
            self.positive_descriptors.clear()
            self.negative_descriptors.clear()
            self.ex_pos_descriptors.clear()
            self.ex_neg_descriptors.clear()
            self.ex_data.clear()

            self.rel_index = None
            self.results = None

            # clear contents of working directory
            shutil.rmtree(self.work_dir)

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import clear [as 别名]

#.........这里部分代码省略.........
            pos = self.positive_descriptors | self.external_positive_descriptors
            neg = self.negative_descriptors | self.external_negative_descriptors

            if not pos:
                raise RuntimeError("Did not find at least one positive "
                                   "adjudication.")

            self._log.debug("Ranking working set with %d pos and %d neg total "
                            "examples.", len(pos), len(neg))
            element_probability_map = self.rel_index.rank(pos, neg)

            if self.results is None:
                self.results = IqrResultsDict()
            self.results.update(element_probability_map)

            # Force adjudicated positives and negatives to be probability 1 and
            # 0, respectively, since we want to control where they show up in
            # our results view.
            # - Not all pos/neg descriptors may be in our working index.
            for d in pos:
                if d in self.results:
                    self.results[d] = 1.0
            for d in neg:
                if d in self.results:
                    self.results[d] = 0.0

    def reset(self):
        """ Reset the IQR Search state

        No positive adjudications, reload original feature data

        """
        with self.lock:
            self.working_index.clear()
            self._wi_seeds_used.clear()
            self.positive_descriptors.clear()
            self.negative_descriptors.clear()
            self.external_positive_descriptors.clear()
            self.external_negative_descriptors.clear()

            self.rel_index = None
            self.results = None

    ###########################################################################
    # I/O Methods

    # I/O Constants. These should not be changed.
    STATE_ZIP_COMPRESSION = zipfile.ZIP_DEFLATED
    STATE_ZIP_FILENAME = "iqr_state.json"

    def get_state_bytes(self):
        """
        Get a byte representation of the current descriptor and adjudication
        state of this session.

        This does not encode current results or the relevancy index's state, but
        these can be reproduced with this state.

        :return: State representation bytes
        :rtype: bytes

        """
        def d_set_to_list(d_set):
            # Convert set of descriptors to list of tuples:
            #   [..., (uuid, type, vector), ...]
            return [(d.uuid(), d.type(), d.vector().tolist()) for d in d_set]

# 需要导入模块: from smqtk.representation.descriptor_index.memory import MemoryDescriptorIndex [as 别名]
# 或者: from smqtk.representation.descriptor_index.memory.MemoryDescriptorIndex import clear [as 别名]

#.........这里部分代码省略.........
            self.negative_descriptors.update(new_negatives)
            self.negative_descriptors.difference_update(un_negatives)
            self.negative_descriptors.difference_update(new_positives)

    def update_working_index(self, nn_index):
        """
        Initialize or update our current working index using the given
        :class:`.NearestNeighborsIndex` instance given our current positively
        labeled descriptor elements.

        We only query from the index for new positive elements since the last
        update or reset.

        :param nn_index: :class:`.NearestNeighborsIndex` to query from.
        :type nn_index: smqtk.algorithms.NearestNeighborsIndex

        :raises RuntimeError: There are no positive example descriptors in this
            session to use as a basis for querying.

        """
        if len(self.positive_descriptors) <= 0:
            raise RuntimeError("No positive descriptors to query the neighbor "
                               "index with.")

        # Not clearing working index because this step is intended to be
        # additive.
        updated = False

        # adding to working index
        for p in self.positive_descriptors:
            if p.uuid() not in self._wi_seeds_used:
                self._log.info("Querying neighbors to: %s", p)
                self.working_index.add_many_descriptors(
                    nn_index.nn(p, n=self.pos_seed_neighbors)[0]
                )
                self._wi_seeds_used.add(p.uuid())
                updated = True

        # Make new relevancy index
        if updated:
            self._log.info("Creating new relevancy index over working index.")
            #: :type: smqtk.algorithms.relevancy_index.RelevancyIndex
            self.rel_index = plugin.from_plugin_config(self.rel_index_config,
                                                       get_relevancy_index_impls())
            self.rel_index.build_index(self.working_index.iterdescriptors())

    def refine(self):
        """ Refine current model results based on current adjudication state

        :raises RuntimeError: No working index has been initialized.
            :meth:`update_working_index` should have been called after
            adjudicating some positive examples.
        :raises RuntimeError: There are no adjudications to run on. We must
            have at least one positive adjudication.

        """
        with self.lock:
            if not self.rel_index:
                raise RuntimeError("No relevancy index yet. Must not have "
                                   "initialized session (no working index).")

            # fuse pos/neg adjudications + added positive data descriptors
            pos = self.positive_descriptors
            neg = self.negative_descriptors

            if not pos:
                raise RuntimeError("Did not find at least one positive "
                                   "adjudication.")

            element_probability_map = self.rel_index.rank(pos, neg)

            if self.results is None:
                self.results = IqrResultsDict()
            self.results.update(element_probability_map)

            # Force adjudicated positives and negatives to be probability 1 and
            # 0, respectively, since we want to control where they show up in
            # our results view.
            # - Not all pos/neg descriptors may be in our working index.
            for d in pos:
                if d in self.results:
                    self.results[d] = 1.0
            for d in neg:
                if d in self.results:
                    self.results[d] = 0.0

    def reset(self):
        """ Reset the IQR Search state

        No positive adjudications, reload original feature data

        """
        with self.lock:
            self.working_index.clear()
            self._wi_seeds_used.clear()
            self.positive_descriptors.clear()
            self.negative_descriptors.clear()

            self.rel_index = None
            self.results = None