Python factory.repo_unit_association_query_manager函数代码示例

def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False):
    """
    Performs a repo unit association query. This is split apart so we can have
    custom mixins with different signatures.
    """
    try:
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        # Use a get_units as_generator here and cast to a list later, if necessary.
        units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True)

        # Transfer object generator.
        def _transfer_object_generator():
            unit_key_fields_cache = {}
            for u in units:
                type_id = u['unit_type_id']
                if type_id not in unit_key_fields_cache:
                    fields = units_controller.get_unit_key_fields_for_type(type_id)
                    unit_key_fields_cache[type_id] = fields

                yield common_utils.to_plugin_associated_unit(u, type_id,
                                                             unit_key_fields_cache[type_id])

        if as_generator:
            return _transfer_object_generator()

        # Maintain legacy behavior by default.
        return list(_transfer_object_generator())

    except Exception, e:
        _logger.exception(
            'Exception from server requesting all content units for repository [%s]' % repo_id)
        raise exception_class(e), None, sys.exc_info()[2]

    def test_migrate_category(self):
        # Setup
        orig_cat_id = add_unit('c1', self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY)

        associate_unit(orig_cat_id, self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY)
        associate_unit(orig_cat_id, self.dest_repo_id, ids.TYPE_ID_PKG_CATEGORY)

        # Test
        self.migration.migrate()

        group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_CATEGORY)
        all_cats = group_coll.find({}).sort('repo_id', 1)
        self.assertEqual(2, all_cats.count())

        dest_cat = all_cats[0] # ordered by ID, this will be first
        self.assertEqual(dest_cat['id'], 'c1')
        self.assertEqual(dest_cat['repo_id'], self.dest_repo_id)

        source_cat = all_cats[1]
        self.assertEqual(source_cat['id'], 'c1')
        self.assertEqual(source_cat['repo_id'], self.source_repo_id)

        # Verify the associations
        query_manager = factory.repo_unit_association_query_manager()

        source_units = query_manager.get_units(self.source_repo_id)
        self.assertEqual(1, len(source_units))
        self.assertEqual(source_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY)
        self.assertEqual(source_units[0]['unit_id'], source_cat['_id'])

        dest_units = query_manager.get_units(self.dest_repo_id)
        self.assertEqual(1, len(dest_units))
        self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY)
        self.assertEqual(dest_units[0]['unit_id'], dest_cat['_id'])

    def resolve_dependencies_by_criteria(self, repo_id, criteria, options):
        """
        Calculates dependencies for units in the given repositories. The
        repository's importer is used to perform the calculation. The units
        to resolve dependencies for are calculated by applying the given
        criteria against the repository.

        @param repo_id: identifies the repository
        @type  repo_id: str

        @param criteria: used to determine which units to resolve dependencies for
        @type  criteria: UnitAssociationCriteria

        @param options: dict of options to pass the importer to drive the resolution
        @type  options: dict

        @return: report from the plugin
        @rtype:  object
        """
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        units = association_query_manager.get_units(repo_id, criteria=criteria)

        # The bulk of the validation will be done in the chained call below

        return self.resolve_dependencies_by_units(repo_id, units, options)

def calculate_associated_type_ids(source_repo_id, associated_units):
    if associated_units is not None:
        associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units])
    else:
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        associated_unit_type_ids = association_query_manager.unit_type_ids_for_repo(source_repo_id)
    return associated_unit_type_ids

    def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id,
                 association_owner_type, association_owner_id):
        """
        :param source_repo_id: ID of the repository from which units are being copied
        :type  source_repo_id: str
        :param dest_repo_id: ID of the repository into which units are being copied
        :type  dest_repo_id: str
        :param source_importer_id: ID of the importer on the source repository
        :type  source_importer_id: str
        :param dest_importer_id:  ID of the importer on the destination repository
        :type  dest_importer_id: str
        :param association_owner_type: distinguishes the owner when creating an
               association through this conduit
        :type  association_owner_type: str
        :param association_owner_id: specific ID of the owner when creating an
               association through this conduit
        :type  association_owner_id: str
        """
        ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
        RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
        SearchUnitsMixin.__init__(self, ImporterConduitException)
        AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id)

        self.source_repo_id = source_repo_id
        self.dest_repo_id = dest_repo_id

        self.source_importer_id = source_importer_id
        self.dest_importer_id = dest_importer_id

        self.association_owner_type = association_owner_type
        self.association_owner_id = association_owner_id

        self.__association_manager = manager_factory.repo_unit_association_manager()
        self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
        self.__importer_manager = manager_factory.repo_importer_manager()

def do_get_repo_units(repo_id, criteria, exception_class):
    """
    Performs a repo unit association query. This is split apart so we can have
    custom mixins with different signatures.
    """
    try:
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        units = association_query_manager.get_units(repo_id, criteria=criteria)

        all_units = []

        # Load all type definitions in use so we don't hammer the database
        unique_type_defs = set([u['unit_type_id'] for u in units])
        type_defs = {}
        for def_id in unique_type_defs:
            type_def = types_db.type_definition(def_id)
            type_defs[def_id] = type_def

        # Convert to transfer object
        for unit in units:
            type_id = unit['unit_type_id']
            u = common_utils.to_plugin_unit(unit, type_defs[type_id])
            all_units.append(u)

        return all_units

    except Exception, e:
        _LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id)
        raise exception_class(e), None, sys.exc_info()[2]

    def _get_total(self, id_list=None, ignore_filter=False):
        """
        Return the total number of units that are processed by this step.
        This is used generally for progress reporting.  The value returned should not change
        during the processing of the step.

        :param id_list: List of type ids to get the total count of
        :type id_list: list of str
        :param ignore_filter: Ignore the association filter and get all units of the given types
        :type ignore_filter: bool
        """
        if id_list is None:
            id_list = self.unit_type
        total = 0
        types_to_query = set(id_list).difference(self.skip_list)
        if not ignore_filter and self.association_filters:
            # We are copying using a filter so we have to get everything
            new_filter = copy.deepcopy(self.association_filters)
            new_filter['unit_type_id'] = {'$in': list(types_to_query)}
            criteria = Criteria(filters=new_filter)
            association_query_manager = manager_factory.repo_unit_association_query_manager()
            units_cursor = association_query_manager.find_by_criteria(criteria)
            total = units_cursor.count()
        else:
            for type_id in types_to_query:
                total += self.parent.repo.content_unit_counts.get(type_id, 0)
        return total

def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False):
    """
    Performs a repo unit association query. This is split apart so we can have
    custom mixins with different signatures.
    """
    try:
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        # Use a get_units as_generator here and cast to a list later, if necessary.
        units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True)

        # Load all type definitions so we don't hammer the database.
        type_defs = dict((t['id'], t) for t in types_db.all_type_definitions())

        # Transfer object generator.
        def _transfer_object_generator():
            for u in units:
                yield common_utils.to_plugin_associated_unit(u, type_defs[u['unit_type_id']])

        if as_generator:
            return _transfer_object_generator()

        # Maintain legacy behavior by default.
        return list(_transfer_object_generator())

    except Exception, e:
        _LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id)
        raise exception_class(e), None, sys.exc_info()[2]

 def verify(self, num_units=PluginTestBase.NUM_UNITS):
     # repository
     manager = managers.repo_query_manager()
     manager.get_repository(self.REPO_ID)
     # importer
     manager = managers.repo_importer_manager()
     importer = manager.get_importer(self.REPO_ID)
     manifest_url = importer['config'][constants.MANIFEST_URL_KEYWORD]
     self.assertTrue(manifest_url.endswith('%s/manifest.json.gz' % self.REPO_ID))
     # distributor
     manager = managers.repo_distributor_manager()
     manager.get_distributor(self.REPO_ID, FAKE_DISTRIBUTOR)
     self.assertRaises(MissingResource, manager.get_distributor, self.REPO_ID, constants.HTTP_DISTRIBUTOR)
     # check units
     manager = managers.repo_unit_association_query_manager()
     units = manager.get_units(self.REPO_ID)
     units = dict([(u['metadata']['N'], u) for u in units])
     self.assertEqual(len(units), num_units)
     for n in range(0, num_units):
         unit = units[n]
         unit_id = self.UNIT_ID % n
         metadata = unit['metadata']
         storage_path = metadata['_storage_path'].replace('//', '/')
         self.assertEqual(unit['unit_type_id'], self.UNIT_TYPE_ID)
         self.assertEqual(unit['repo_id'], self.REPO_ID)
         self.assertEqual(unit['owner_id'], constants.HTTP_IMPORTER)
         file_path = '.'.join((unit_id, self.UNIT_TYPE_ID))
         self.assertEqual(storage_path, os.path.join(self.childfs, 'content', file_path))
         self.assertTrue(os.path.exists(storage_path))
         fp = open(storage_path)
         content = fp.read()
         fp.close()
         self.assertEqual(content, unit_id)

    def POST(self, repo_id):
        # Params
        params = self.params()
        query = params.get('criteria', None)

        repo_query_manager = manager_factory.repo_query_manager()
        repo = repo_query_manager.find_by_id(repo_id)
        if repo is None:
            raise exceptions.MissingResource(repo_id=repo_id)

        if query is None:
            raise exceptions.MissingValue(['criteria'])

        try:
            criteria = UnitAssociationCriteria.from_client_input(query)
        except:
            _LOG.exception('Error parsing association criteria [%s]' % query)
            raise exceptions.PulpDataException(), None, sys.exc_info()[2]

        # Data lookup
        manager = manager_factory.repo_unit_association_query_manager()
        if criteria.type_ids is not None and len(criteria.type_ids) == 1:
            type_id = criteria.type_ids[0]
            units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
        else:
            units = manager.get_units_across_types(repo_id, criteria=criteria)

        return self.ok(units)

    def _add_repo_memberships(units, type_id):
        """
        For a list of units, find what repos each is a member of and add a list
        of repo_ids to each unit.

        :param units:   list of unit documents
        :type  units:   list of dicts
        :param type_id: content type id
        :type  type_id: str
        :return:    same list of units that was passed in, only for convenience.
                    units are modified in-place
        """
        # quick return if there is nothing to do
        if not units:
            return units

        unit_ids = [unit["_id"] for unit in units]
        criteria = Criteria(
            filters={"unit_id": {"$in": unit_ids}, "unit_type_id": type_id}, fields=("repo_id", "unit_id")
        )
        associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria)
        unit_ids = None
        criteria = None
        association_map = {}
        for association in associations:
            association_map.setdefault(association["unit_id"], set()).add(association["repo_id"])

        for unit in units:
            unit["repository_memberships"] = list(association_map.get(unit["_id"], []))
        return units

    def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id):
        """
        :param source_repo_id: ID of the repository from which units are being copied
        :type  source_repo_id: str
        :param dest_repo_id: ID of the repository into which units are being copied
        :type  dest_repo_id: str
        :param source_importer_id: ID of the importer on the source repository
        :type  source_importer_id: str
        :param dest_importer_id:  ID of the importer on the destination repository
        :type  dest_importer_id: str
        """
        ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id)
        RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException)
        SearchUnitsMixin.__init__(self, ImporterConduitException)
        AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id)

        self.source_repo_id = source_repo_id
        self.dest_repo_id = dest_repo_id

        self.source_importer_id = source_importer_id
        self.dest_importer_id = dest_importer_id

        self.__association_manager = manager_factory.repo_unit_association_manager()
        self.__association_query_manager = manager_factory.repo_unit_association_query_manager()
        self.__importer_manager = manager_factory.repo_importer_manager()

    def _generate_response(cls, query, options, *args, **kwargs):
        """
        Perform the database query using the given search data, and return the resuls as a JSON
        serialized HttpReponse object.

        This overrides the base class so we can validate repo existance and to choose the search
        method depending on how many unit types we are dealing with.

        :param query: The criteria that should be used to search for objects
        :type  query: dict
        :param options: additional options for including extra data
        :type  options: dict

        :return:      The serialized search results in an HttpReponse
        :rtype:       django.http.HttpResponse
        """
        repo_id = kwargs.get('repo_id')
        model.Repository.objects.get_repo_or_missing_resource(repo_id)
        criteria = UnitAssociationCriteria.from_client_input(query)
        manager = manager_factory.repo_unit_association_query_manager()
        if criteria.type_ids is not None and len(criteria.type_ids) == 1:
            type_id = criteria.type_ids[0]
            units = manager.get_units_by_type(repo_id, type_id, criteria=criteria)
        else:
            units = manager.get_units(repo_id, criteria=criteria)
        for unit in units:
            content.remap_fields_with_serializer(unit['metadata'])
        return generate_json_response_with_pulp_encoder(units)

def load_associated_units(source_repo_id, criteria):
    criteria.association_fields = None

    # Retrieve the units to be associated
    association_query_manager = manager_factory.repo_unit_association_query_manager()
    associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria)

    return associate_us

    def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True):
        """
        Unassociate units that are matched by the given criteria.

        :param repo_id:        identifies the repo
        :type  repo_id:        str
        :param criteria:
        :param owner_type:     category of the caller who created the association
        :type  owner_type:     str
        :param owner_id:       identifies the call who created the association
        :type  owner_id:       str
        :param notify_plugins: if true, relevant plugins will be informed of the removal
        :type  notify_plugins: bool
        """
        association_query_manager = manager_factory.repo_unit_association_query_manager()
        unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria)

        if len(unassociate_units) == 0:
            return {}

        unit_map = {}  # maps unit_type_id to a list of unit_ids

        for unit in unassociate_units:
            id_list = unit_map.setdefault(unit['unit_type_id'], [])
            id_list.append(unit['unit_id'])

        collection = RepoContentUnit.get_collection()
        repo_manager = manager_factory.repo_manager()

        for unit_type_id, unit_ids in unit_map.items():
            spec = {'repo_id': repo_id,
                    'unit_type_id': unit_type_id,
                    'unit_id': {'$in': unit_ids}
                    }
            collection.remove(spec, safe=True)

            unique_count = sum(
                1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists(
                    repo_id, unit_id, unit_type_id))
            if not unique_count:
                continue

            repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count)

        repo_manager.update_last_unit_removed(repo_id)

        # Convert the units into transfer units. This happens regardless of whether or not
        # the plugin will be notified as it's used to generate the return result,
        unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units)
        transfer_units = create_transfer_units(unassociate_units, unit_type_ids)

        if notify_plugins:
            remove_from_importer(repo_id, transfer_units)

        # Match the return type/format as copy
        serializable_units = [u.to_id_dict() for u in transfer_units]

        return {'units_successful': serializable_units}