Python repository.associate_single_unit函数代码示例

    def process_main(self, item):
        """
        This method gets called with each Unit that was downloaded from the parent step. It moves
        each Unit's files into permanent storage, and saves each Unit into the database and into the
        repository.

        :param item: The Image to save in Pulp
        :type  item: pulp_docker.plugins.models.Image
        """
        with open(os.path.join(self.get_working_dir(), item.image_id, 'json')) as json_file:
            metadata = json.load(json_file)
        # at least one old docker image did not have a size specified in
        # its metadata
        size = metadata.get('Size')
        # an older version of docker used a lowercase "p"
        parent = metadata.get('parent', metadata.get('Parent'))
        item.parent_id = parent
        item.size = size

        tmp_dir = os.path.join(self.get_working_dir(), item.image_id)
        item.save()
        for name in os.listdir(tmp_dir):
            path = os.path.join(tmp_dir, name)
            item.import_content(path, location=os.path.basename(path))

        repo_controller.associate_single_unit(self.get_repo().repo_obj, item)

    def upload_unit(self, repo, type_id, unit_key, metadata, file_path, conduit, config):
        """
        Handles a user request to upload a unit into a repository. This call
        should use the data provided to add the unit as if it were synchronized
        from an external source. This includes:

        * Initializing the unit through the conduit which populates the final
          destination of the unit.
        * Moving the unit from the provided temporary location into the unit's
          final destination.
        * Saving the unit in Pulp, which both adds the unit to Pulp's database and
          associates it to the repository.

        This call may be invoked for either units that do not already exist as
        well as re-uploading an existing unit.

        The metadata parameter is variable in its usage. In some cases, the
        unit may be almost exclusively metadata driven in which case the contents
        of this parameter will be used directly as the unit's metadata. In others,
        it may function to remove the importer's need to derive the unit's metadata
        from the uploaded unit file. In still others, it may be extraneous
        user-specified information that should be merged in with any derived
        unit metadata.

        Depending on the unit type, it is possible that this call will create
        multiple units within Pulp. It is also possible that this call will
        create one or more relationships between existing units.

        :param repo:      metadata describing the repository
        :type  repo:      pulp.plugins.model.Repository
        :param type_id:   type of unit being uploaded
        :type  type_id:   str
        :param unit_key:  identifier for the unit, specified by the user
        :type  unit_key:  dict
        :param metadata:  any user-specified metadata for the unit
        :type  metadata:  dict
        :param file_path: path on the Pulp server's filesystem to the temporary location of the
                          uploaded file; may be None in the event that a unit is comprised entirely
                          of metadata and has no bits associated
        :type  file_path: str
        :param conduit:   provides access to relevant Pulp functionality
        :type  conduit:   pulp.plugins.conduits.unit_add.UnitAddConduit
        :param config:    plugin configuration for the repository
        :type  config:    pulp.plugins.config.PluginCallConfiguration
        :return:          A dictionary describing the success or failure of the upload. It must
                          contain the following keys:
                            'success_flag': bool. Indicates whether the upload was successful
                            'summary':      json-serializable object, providing summary
                            'details':      json-serializable object, providing details
        :rtype:           dict
        """
        package = models.Package.from_archive(file_path)
        try:
            package.save_and_import_content(file_path)
        except NotUniqueError:
            package = package.__class__.objects.get(**package.unit_key)

        repo_controller.associate_single_unit(repo.repo_obj, package)

        return {'success_flag': True, 'summary': {}, 'details': {}}

def _handle_yum_metadata_file(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the upload for a yum repository metadata file.

    :type  repo: pulp.server.db.model.Repository
    :type  type_id: str
    :type  unit_key: dict
    :type  metadata: dict or None
    :type  file_path: str
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit
    :type  config: pulp.plugins.config.PluginCallConfiguration
    """

    # Validate the user specified data by instantiating the model
    model_data = dict()
    model_data.update(unit_key)
    if metadata:
        model_data.update(metadata)

    # Replicates the logic in yum/sync.py.import_unknown_metadata_files.
    # The local_path variable is removed since it's not included in the metadata when
    # synchronized.
    file_relative_path = model_data.pop('local_path')

    translated_data = models.YumMetadataFile.SERIALIZER().from_representation(model_data)

    model = models.YumMetadataFile(**translated_data)
    model.set_content(file_relative_path)
    model.save()

    # Move the file to its final storage location in Pulp
    repo_controller.associate_single_unit(conduit.repo, model)

def associate_copy_for_repo(unit, dest_repo, set_content=False):
    """
    Associate a unit where it is required to make a copy of the unit first, and where the unit key
    includes the repo ID.

    :param unit:            Unit to be copied
    :type  unit:            pulp_rpm.plugins.db.models.Package
    :param dest_repo:       destination repo
    :type  dest_repo:       pulp.server.db.model.Repository
    :param set_content:     if True, the set_unit() method will be called on the new unit. Default
                            is False.
    :type  set_content:     bool

    :return:    new unit that was saved and associated
    :rtype:     pulp_rpm.plugins.db.models.Package
    """
    new_unit = unit.clone()
    new_unit.repo_id = dest_repo.repo_id

    try:
        new_unit.save()
    except mongoengine.NotUniqueError:
        # It is possible that a previous copy exists as an orphan, in which case it can safely
        # be deleted and replaced with this new version.
        _LOGGER.debug(_('replacing pre-existing copy of %(u)s' % {'u': new_unit}))
        new_unit.__class__.objects.filter(**new_unit.unit_key).delete()
        new_unit.save()

    if set_content:
        new_unit.set_storage_path(os.path.basename(unit._storage_path))
        new_unit.safe_import_content(unit._storage_path)

    repo_controller.associate_single_unit(repository=dest_repo, unit=new_unit)
    return new_unit

    def download_succeeded(self, report):
        """
        This is the callback that we will get from the downloader library when it succeeds in
        downloading a file. This method will check to see if we are in the ISO downloading stage,
        and if we are, it will add the new ISO to the database.

        :param report: The report of the file we downloaded
        :type  report: nectar.report.DownloadReport
        """
        # If we are in the isos stage, then this must be one of our ISOs.
        if self.progress_report.state == self.progress_report.STATE_ISOS_IN_PROGRESS:
            # This will update our bytes downloaded
            self.download_progress(report)
            iso = report.data
            iso.set_storage_path(os.path.basename(report.destination))
            try:
                if self._validate_downloads:
                    iso.validate_iso(report.destination)
                iso.save_and_import_content(report.destination)
                repo_controller.associate_single_unit(self.sync_conduit.repo, iso)

                # We can drop this ISO from the url --> ISO map
                self.progress_report.num_isos_finished += 1
                self.progress_report.update_progress()
            except ValueError:
                self.download_failed(report)

    def download_succeeded(self, report):
        """
        The callback when a download succeeds.

        :param report: the report for the succeeded download.
        :type  report: nectar.report.DownloadReport
        """
        model = report.data

        try:
            self._verify_size(model, report)
            self._verify_checksum(model, report)
        except verification.VerificationException:
            # The verify methods populates the error details of the progress report.
            # There is also no need to clean up the bad file as the sync will blow away
            # the temp directory after it finishes. Simply punch out so the good unit
            # handling below doesn't run.
            return
        except verification.InvalidChecksumType:
            return

        # these are the only types we store repo metadata snippets on in the DB
        if isinstance(model, (models.RPM, models.SRPM)):
            self.metadata_files.add_repodata(model)

        purge.remove_unit_duplicate_nevra(model, self.sync_conduit.repo)

        model.set_content(report.destination)
        model.save()

        repo_controller.associate_single_unit(self.sync_conduit.repo, model)

        # TODO consider that if an exception occurs before here maybe it shouldn't call success?
        self.progress_report['content'].success(model)
        self.sync_conduit.set_progress(self.progress_report)

    def _import_manifest(conduit, unit, dest_repo):
        """
        Import a Manifest and its referenced Blobs.

        :param conduit:   provides access to relevant Pulp functionality
        :type  conduit:   pulp.plugins.conduits.unit_import.ImportUnitConduit
        :param unit:      The Manifest to import
        :type  unit:      pulp_docker.plugins.Model.Manifest
        :param dest_repo: The destination repository that the Manifest is being imported to.
        :type  dest_repo: pulp.server.db.model.Repository
        :return:          list of Units that were copied to the destination repository
        :rtype:           list
        """
        units_added = set()

        # Add manifests and catalog referenced blobs
        blob_digests = set()
        repository.associate_single_unit(dest_repo, unit)
        units_added.add(unit)
        for layer in unit.fs_layers:
            blob_digests.add(layer.blob_sum)

        # Add referenced blobs
        for blob in models.Blob.objects.filter(digest__in=sorted(blob_digests)):
            units_added |= set(DockerImporter._import_blob(conduit, blob, dest_repo))

        return units_added

def _get_and_save_file_units(filename, processing_function, tag, conduit, repo):
    """
    Given a comps.xml file, this method decides which groups/categories to get and saves
    the parsed units.

    :param filename:  open file-like object containing metadata
    :type  filename:  file

    :param processing_function:  method to use for generating the units
    :type  processing_function:  function

    :param tag:  XML tag that identifies each unit
    :type  tag:  str

    :param conduit:  provides access to relevant Pulp functionality
    :type  conduit:  pulp.plugins.conduits.upload.UploadConduit

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository
    """
    repo_id = repo.repo_id
    process_func = functools.partial(processing_function, repo_id)
    package_info_generator = packages.package_list_generator(filename, tag, process_func)
    for model in package_info_generator:
        try:
            model.save()
        except NotUniqueError:
            model = model.__class__.objects.filter(**model.unit_key).first()

        repo_controller.associate_single_unit(repo, model)

def _associate_unit(dest_repo, unit):
    """
    Associate one particular unit with the destination repository. There are
    behavioral exceptions based on type:

    Group, Category, Environment and Yum Metadata File units need to have their "repo_id"
    attribute set.

    RPMs are convenient to do all as one block, for the purpose of dependency
    resolution. So this method skips RPMs and lets them be done together by
    other means

    :param dest_repo:       destination repo
    :type  dest_repo:       pulp.server.db.model.Repository

    :param unit:            Unit to be copied
    :type  unit:            pulp.server.db.model.ContentUnit

    :return:                copied unit
    :rtype:                 pulp.server.db.model.ContentUnit
    """
    if isinstance(unit, (models.PackageGroup, models.PackageCategory, models.PackageEnvironment)):
        return associate_copy_for_repo(unit, dest_repo)
    elif isinstance(unit, models.RPM):
        # copy will happen in one batch
        return unit
    elif isinstance(unit, models.YumMetadataFile):
        return associate_copy_for_repo(unit, dest_repo, True)
    else:
        repo_controller.associate_single_unit(repository=dest_repo, unit=unit)
        return unit

    def _add_new_module(self, downloader, module):
        """
        Performs the tasks for downloading and saving a new unit in Pulp.

        This method entirely skips modules that are already in the repository.

        :param downloader: downloader instance to use for retrieving the unit
        :type downloader: child of pulp_puppet.plugins.importers.downloaders.base.BaseDownloader

        :param module: module to download and add
        :type  module: pulp_puppet.plugins.db.models.Module
        """
        try:
            # Download the bits
            downloaded_filename = downloader.retrieve_module(self.progress_report, module)

            # Extract the extra metadata into the module
            metadata = metadata_module.extract_metadata(downloaded_filename,
                                                        self.repo.working_dir)

            # Overwrite the author and name
            metadata.update(Module.split_filename(metadata['name']))

            # Create and save the Module
            module = Module.from_metadata(metadata)
            module.set_storage_path(os.path.basename(downloaded_filename))
            try:
                module.save_and_import_content(downloaded_filename)
            except NotUniqueError:
                module = module.__class__.objects.get(**module.unit_key)
            # Associate the module with the repo
            repo_controller.associate_single_unit(self.repo.repo_obj, module)
        finally:
            downloader.cleanup_module(module)

    def import_unknown_metadata_files(self, metadata_files):
        """
        Import metadata files whose type is not known to us. These are any files
        that we are not already parsing.

        :param metadata_files:  object containing access to all metadata files
        :type  metadata_files:  pulp_rpm.plugins.importers.yum.repomd.metadata.MetadataFiles
        """
        for metadata_type, file_info in metadata_files.metadata.iteritems():
            if metadata_type not in metadata_files.KNOWN_TYPES:
                file_path = file_info['local_path']
                checksum_type = file_info['checksum']['algorithm']
                checksum_type = verification.sanitize_checksum_type(checksum_type)
                checksum = file_info['checksum']['hex_digest']
                # Find an existing model
                model = models.YumMetadataFile.objects.filter(
                    data_type=metadata_type,
                    repo_id=self.repo.repo_id).first()
                # If an existing model, use that
                if model:
                    model.checksum = checksum
                    model.checksum_type = checksum_type
                else:
                    # Else, create a  new mode
                    model = models.YumMetadataFile(
                        data_type=metadata_type,
                        repo_id=self.repo.repo_id,
                        checksum=checksum,
                        checksum_type=checksum_type)

                model.set_storage_path(os.path.basename(file_path))
                model.save_and_import_content(file_path)

                # associate/re-associate model to the repo
                repo_controller.associate_single_unit(self.repo, model)

    def process_main(self, item=None):
        """
        given the passed-in unit keys, determine which of them already exist in
        pulp, and save those with the conduit found on the parent.

        :param item: The item to process or none if get_iterator is not defined
        :param item: object or None
        """
        # any units that are already in pulp
        units_we_already_had = set()

        # If available_units was defined in the constructor, let's use it. Otherwise let's use the
        # default of self.parent.available_units
        available_units = self.available_units or self.parent.available_units

        for units_group in misc.paginate(available_units, self.unit_pagination_size):
            # Get this group of units
            query = units_controller.find_units(units_group)

            for found_unit in query:
                units_we_already_had.add(hash(found_unit))
                repo_controller.associate_single_unit(self.get_repo().repo_obj, found_unit)

            for unit in units_group:
                if hash(unit) not in units_we_already_had:
                    self.units_to_download.append(unit)

    def _resolve_new_units(self, existing, wanted):
        """
        Decide what units are needed to be downloaded.

        Filter out units which are already in a repository,
        associate units which are already downloaded,

        :param existing: units which are already in a repository
        :type existing: list of unit keys as namedtuples
        :param wanted: units which should be imported into a repository
        :type wanted: list of unit keys as namedtuples

        :return: list of unit keys to download; empty list if all units are already downloaded
        :rtype:  list of unit keys as namedtuples
        """
        model = plugin_api.get_unit_model_by_id(constants.TYPE_PUPPET_MODULE)
        unit_generator = (model(**unit_tuple._asdict()) for unit_tuple in wanted)
        still_wanted = set(wanted)
        for unit in units_controller.find_units(unit_generator):
            file_exists = unit._storage_path is not None and os.path.isfile(unit._storage_path)
            if file_exists:
                if unit.unit_key_as_named_tuple not in existing:
                    repo_controller.associate_single_unit(self.repo.repo_obj, unit)
                still_wanted.discard(unit.unit_key_as_named_tuple)

        return list(still_wanted)

def _handle_group_category_comps(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the creation of a package group or category.

    If a file was uploaded, treat this as upload of a comps.xml file. If no file was uploaded,
    the process only creates the unit.

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration
    """
    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    if file_path is not None and os.path.getsize(file_path) > 0:
        # uploading a comps.xml
        repo_id = repo.repo_id
        _get_and_save_file_units(file_path, group.process_group_element,
                                 group.GROUP_TAG, conduit, repo_id)
        _get_and_save_file_units(file_path, group.process_category_element,
                                 group.CATEGORY_TAG, conduit, repo_id)
        _get_and_save_file_units(file_path, group.process_environment_element,
                                 group.ENVIRONMENT_TAG, conduit, repo_id)
    else:
        # uploading a package group or package category
        unit_data = {}
        unit_data.update(metadata or {})
        unit_data.update(unit_key or {})
        try:
            unit = model_class(**unit_data)
        except TypeError:
            raise ModelInstantiationError()

        unit.save()

        if file_path:
            unit.set_storage_path(os.path.basename(file_path))
            unit.safe_import_content(file_path)

        repo_controller.associate_single_unit(repo, unit)

def _handle_erratum(repo, type_id, unit_key, metadata, file_path, conduit, config):
    """
    Handles the upload for an erratum. There is no file uploaded so the only
    steps are to save the metadata and optionally link the erratum to RPMs
    in the repository.

    NOTE: For now errata is handled differently than other units. Uploaded erratum should not
    overwrite the existing one if the latter exists, they should be merged. This is only because
    of the way erratum is stored in the MongoDB and it is in `our plans`_ to re-think how to do
    it correctly.

    .. _our plans: https://pulp.plan.io/issues/1803

    :param repo: The repository to import the package into
    :type  repo: pulp.server.db.model.Repository

    :param type_id: The type_id of the package being uploaded
    :type  type_id: str

    :param unit_key: A dictionary of fields to overwrite introspected field values
    :type  unit_key: dict

    :param metadata: A dictionary of fields to overwrite introspected field values, or None
    :type  metadata: dict or None

    :param file_path: The path to the uploaded package
    :type  file_path: str

    :param conduit: provides access to relevant Pulp functionality
    :type  conduit: pulp.plugins.conduits.upload.UploadConduit

    :param config: plugin configuration for the repository
    :type  config: pulp.plugins.config.PluginCallConfiguration
    """
    model_class = plugin_api.get_unit_model_by_id(type_id)
    update_fields_inbound(model_class, unit_key or {})
    update_fields_inbound(model_class, metadata or {})

    unit_data = {}
    unit_data.update(metadata or {})
    unit_data.update(unit_key or {})

    existing_unit = model_class.objects.filter(**unit_key).first()
    new_unit = model_class(**unit_data)

    # Add repo_id to each collection of the pkglist of the new erratum
    for collection in new_unit.pkglist:
        collection['_pulp_repo_id'] = repo.repo_id

    unit = new_unit
    if existing_unit:
        existing_unit.merge_errata(new_unit)
        unit = existing_unit

    unit.save()
    if not config.get_boolean(CONFIG_SKIP_ERRATUM_LINK):
        repo_controller.associate_single_unit(repo, unit)