Python misc.paginate函数代码示例

    def delete_orphan_content_units_by_type(type_id, content_unit_ids=None):
        """
        Delete the orphaned content units for the given content type.
        This method only applies to new style content units that are loaded via entry points

        NOTE: this method deletes the content unit's bits from disk, if applicable.

        :param type_id: id of the content type
        :type type_id: basestring
        :param content_unit_ids: list of content unit ids to delete; None means delete them all
        :type content_unit_ids: iterable or None
        :return: count of units deleted
        :rtype: int
        """
        # get the model matching the type
        content_model = plugin_api.get_unit_model_by_id(type_id)
        if content_unit_ids:
            query_sets = []
            for page in plugin_misc.paginate(content_unit_ids):
                qs = content_model.objects(id__in=page).only('id', '_storage_path')
                query_sets.append(qs)
            content_units = itertools.chain(*query_sets)
        else:
            content_units = content_model.objects.only('id', '_storage_path')

        count = 0

        # Paginate the content units
        for units_group in plugin_misc.paginate(content_units):
            # Build the list of ids to search for an easier way to access units in the group by id
            unit_dict = dict()
            for unit in units_group:
                unit_dict[unit.id] = unit

            id_list = list(unit_dict.iterkeys())

            # Clear the units that are currently associated from unit_dict
            non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
                .distinct('unit_id')
            for non_orphan_id in non_orphan:
                unit_dict.pop(non_orphan_id)

            # Remove the unit, lazy catalog entries, and any content in storage.
            for unit_to_delete in unit_dict.itervalues():
                model.LazyCatalogEntry.objects(
                    unit_id=str(unit_to_delete.id),
                    unit_type_id=str(type_id)
                ).delete()
                unit_to_delete.delete()
                if unit_to_delete._storage_path:
                    OrphanManager.delete_orphaned_file(unit_to_delete._storage_path)
                count += 1

        return count

def get_existing_units(search_dicts, unit_class, repo):
    """
    Get units from the given repository that match the search terms. The unit instances will only
    have their unit key fields populated.

    :param search_dicts:    iterable of dictionaries that should be used to search units
    :type  search_dicts:    iterable
    :param unit_class:      subclass representing the type of unit to search for
    :type  unit_class:      pulp_rpm.plugins.db.models.Package
    :param repo:            repository to search in
    :type  repo:            pulp.server.db.model.Repository

    :return:    generator of unit_class instances with only their unit key fields populated
    :rtype:     generator
    """
    unit_fields = unit_class.unit_key_fields
    for segment in paginate(search_dicts):
        unit_filters = {'$or': list(segment)}
        units_q = mongoengine.Q(__raw__=unit_filters)
        association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)

        for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
                                                              repo_content_unit_q=association_q,
                                                              unit_fields=unit_fields,
                                                              yield_content_unit=True):
            yield result

    def delete_orphan_content_units_by_type(type_id):
        """
        Delete the orphaned content units for the given content type.
        This method only applies to new style content units that are loaded via entry points

        NOTE: this method deletes the content unit's bits from disk, if applicable.

        :param type_id: id of the content type
        :type type_id: basestring
        """
        # get the model matching the type
        content_model = plugin_api.get_unit_model_by_id(type_id)
        content_units = content_model.objects().only('id', 'storage_path')

        # Paginate the content units
        for units_group in plugin_misc.paginate(content_units):
            # Build the list of ids to search for an easier way to access units in the group by id
            unit_dict = dict()
            for unit in units_group:
                unit_dict[unit.id] = unit

            id_list = list(unit_dict.iterkeys())

            # Clear the units that are currently associated from unit_dict
            non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
                .distinct('unit_id')
            for non_orphan_id in non_orphan:
                unit_dict.pop(non_orphan_id)

            # Remove the unit and any references on disk
            for unit_to_delete in unit_dict.itervalues():
                unit_to_delete.delete()
                if unit_to_delete.storage_path:
                    OrphanManager.delete_orphaned_file(unit_to_delete.storage_path)

    def process_main(self, item=None):
        """
        given the passed-in unit keys, determine which of them already exist in
        pulp, and save those with the conduit found on the parent.

        :param item: The item to process or none if get_iterator is not defined
        :param item: object or None
        """
        # any units that are already in pulp
        units_we_already_had = set()

        # If available_units was defined in the constructor, let's use it. Otherwise let's use the
        # default of self.parent.available_units
        available_units = self.available_units or self.parent.available_units

        for units_group in misc.paginate(available_units, self.unit_pagination_size):
            # Get this group of units
            query = units_controller.find_units(units_group)

            for found_unit in query:
                units_we_already_had.add(hash(found_unit))
                repo_controller.associate_single_unit(self.get_repo().repo_obj, found_unit)

            for unit in units_group:
                if hash(unit) not in units_we_already_had:
                    self.units_to_download.append(unit)

def get_unit_model_querysets(repo_id, model_class, repo_content_unit_q=None):
    """
    Return a generator of mongoengine.queryset.QuerySet objects that collectively represent the
    units in the specified repo that are of the type corresponding to the model_class and that
    match the optional query.

    Results are broken up into multiple QuerySet objects, because units are requested by their ID,
    and we do not want to exceed the maximum size for a single query by stuffing too many IDs in one
    QuerySet object.

    You are welcome and encouraged to convert the return value into one generator of ContentUnit
    objects by using itertools.chain()

    :param repo_id:     ID of the repo whose units should be queried
    :type  repo_id:     str
    :param model_class: a subclass of ContentUnit that defines a unit model
    :type  model_class: pulp.server.db.model.ContentUnit
    :param repo_content_unit_q: any additional filters that should be applied to the
                                RepositoryContentUnit search
    :type  repo_content_unit_q: mongoengine.Q

    :return:    generator of mongoengine.queryset.QuerySet
    :rtype:     generator
    """
    for chunk in paginate(get_associated_unit_ids(repo_id,
                                                  model_class._content_type_id.default,
                                                  repo_content_unit_q)):
        yield model_class.objects(id__in=chunk)

def find_units(units, pagination_size=50):
    """
    Query for units matching the unit key fields of an iterable of ContentUnit objects.

    This requires that all the ContentUnit objects are of the same content type.

    :param units: Iterable of content units with the unit key fields specified.
    :type units: iterable of pulp.server.db.model.ContentUnit
    :param pagination_size: How large a page size to use when querying units.
    :type pagination_size: int (default 50)

    :returns: unit models that pulp already knows about.
    :rtype: Generator of pulp.server.db.model.ContentUnit
    """
    # get the class from the first unit
    model_class = None

    for units_group in misc.paginate(units, pagination_size):
        q_object = mongoengine.Q()
        # Build a query for the units in this group
        for unit in units_group:
            if model_class is None:
                model_class = unit.__class__

            # Build the query for all the units, the | operator here
            # creates the equivalent of a mongo $or of all the unit keys
            unit_q_obj = mongoengine.Q(**unit.unit_key)
            q_object = q_object | unit_q_obj

        # Get this group of units
        query = model_class.objects(q_object)

        for found_unit in query:
            yield found_unit

    def test_empty_list(self):
        ret = misc.paginate([], 3)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [])

    def test_generator_one_page(self):
        iterable = (x for x in range(10))
        ret = misc.paginate(iterable, 100)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [tuple(range(10))])

    def test_list(self):
        iterable = list(range(10))
        ret = misc.paginate(iterable, 3)

        self.assertTrue(inspect.isgenerator(ret))

        pieces = list(ret)

        self.assertEqual(pieces, [(0,1,2), (3,4,5), (6,7,8), (9,)])

def disassociate_units(repository, unit_iterable):
    """
    Disassociate all units in the iterable from the repository

    :param repository: The repository to update.
    :type repository: pulp.server.db.model.Repository
    :param unit_iterable: The units to disassociate from the repository.
    :type unit_iterable: iterable of pulp.server.db.model.ContentUnit
    """
    for unit_group in misc.paginate(unit_iterable):
        unit_id_list = [unit.id for unit in unit_group]
        qs = model.RepositoryContentUnit.objects(
            repo_id=repository.repo_id, unit_id__in=unit_id_list)
        qs.delete()

def get_existing_units(search_dicts, unit_fields, unit_type, search_method):
    """

    :param search_dicts:
    :param unit_fields:
    :param unit_type:
    :param search_method:
    :return:    generator of Units
    """
    for segment in paginate(search_dicts):
        unit_filters = {'$or': list(segment)}
        criteria = UnitAssociationCriteria([unit_type], unit_filters=unit_filters,
                                           unit_fields=unit_fields, association_fields=[])
        for result in search_method(criteria):
            yield result

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of pulp_rpm.plugins.models.RPM
        :type  units:   iterable

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            for unit in segment:
                for require in unit.requires or []:
                    yield Requirement(**require)

    def open_cursors(unit_ids):
        """
        Get a generator of unit cursors.

        :param unit_ids: A dictionary of unit_ids keyed by type_id.
        :type unit_ids: dict
        :return: A list of open cursors.
        :rtype: generator
        """
        for type_id, id_list in unit_ids.items():
            for page in paginate(id_list):
                query = {'_id': {'$in': page}}
                collection = type_units_collection(type_id)
                cursor = collection.find(query)
                yield cursor

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of RPMs for which a query should be performed to
                        retrieve their Requires entries.
        :type  units:   iterable of pulp.plugins.model.Unit

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            unit_ids = [unit.id for unit in segment]
            fields = ['requires', 'id']
            for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
                for require in result.requires or []:
                    yield Requirement(**require)

    def get_requirements(self, units):
        """
        For an iterable of RPM Units, return a generator of Require() instances that
        represent the requirements for those RPMs.

        :param units:   iterable of pulp_rpm.plugins.models.RPM
        :type  units:   iterable

        :return:    generator of Require() instances
        :rtype:     generator
        """
        for segment in paginate(units):
            unit_ids = [unit.id for unit in segment]
            # The "requires" field is required by this workflow, but is not populated on the
            # incoming "units". Thus we need to re-fetch units with that field.
            fields = ['requires', 'id']
            for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
                for require in result.requires or []:
                    yield Requirement(**require)