Python util.encode_unicode函数代码示例

 def unit_key_str(unit_key_dict):
     """
     Converts the unit key dict form into a single string that can be
     used as the key in a dict lookup.
     """
     template = '%s-%s-%s'
     return template % (encode_unicode(unit_key_dict['name']),
                        encode_unicode(unit_key_dict['version']),
                        encode_unicode(unit_key_dict['author']))

    def test_create_i18n(self):
        # Setup
        i18n_text = 'Brasília'

        # Test
        self.manager.create_repo('repo-i18n', display_name=i18n_text, description=i18n_text)

        # Verify
        repo = Repo.get_collection().find_one({'id': 'repo-i18n'})
        self.assertTrue(repo is not None)
        self.assertEqual(encode_unicode(repo['display_name']), i18n_text)
        self.assertEqual(encode_unicode(repo['description']), i18n_text)

    def deserialize(cls, data):
        """
        Deserialize the data returned from a serialize call into a call request
        @param data: serialized call request
        @type data: dict
        @return: deserialized call request
        @rtype: CallRequest
        """

        if data is None:
            return None

        constructor_kwargs = dict(data)
        constructor_kwargs.pop('callable_name') # added for search

        for key, value in constructor_kwargs.items():
            constructor_kwargs[encode_unicode(key)] = constructor_kwargs.pop(key)

        try:
            for field in cls.pickled_fields:
                constructor_kwargs[field] = pickle.loads(data[field].encode('ascii'))

        except Exception, e:
            _LOG.exception(e)
            return None

    def _download_file(self, url, destination):
        """
        Downloads the content at the given URL into the given destination.
        The object passed into destination must have a method called "update"
        that accepts a single parameter (the buffer that was read).

        :param url: location to download
        :type  url: str

        :param destination: object
        @return:
        """
        curl = self._create_and_configure_curl()
        url = encode_unicode(url) # because of how the config is stored in pulp

        curl.setopt(pycurl.URL, url)
        curl.setopt(pycurl.WRITEFUNCTION, destination.update)
        curl.perform()
        status = curl.getinfo(curl.HTTP_CODE)
        curl.close()

        if status == 401:
            raise exceptions.UnauthorizedException(url)
        elif status == 404:
            raise exceptions.FileNotFoundException(url)
        elif status != 200:
            raise exceptions.FileRetrievalException(url)

    def __init__(self, sync_conduit, config):
        self.sync_conduit = sync_conduit
        self._remove_missing_units = config.get(constants.CONFIG_REMOVE_MISSING_UNITS, default=False)
        self._repo_url = encode_unicode(config.get(constants.CONFIG_FEED_URL))
        self._validate_downloads = config.get(constants.CONFIG_VALIDATE_DOWNLOADS, default=True)

        # Cast our config parameters to the correct types and use them to build a Downloader
        max_speed = config.get(constants.CONFIG_MAX_SPEED)
        if max_speed is not None:
            max_speed = float(max_speed)
        num_threads = config.get(constants.CONFIG_NUM_THREADS)
        if num_threads is not None:
            num_threads = int(num_threads)
        else:
            num_threads = constants.DEFAULT_NUM_THREADS
        downloader_config = {
            'max_speed': max_speed, 'num_threads': num_threads,
            'ssl_client_cert': config.get(constants.CONFIG_SSL_CLIENT_CERT),
            'ssl_client_key': config.get(constants.CONFIG_SSL_CLIENT_KEY),
            'ssl_ca_cert': config.get(constants.CONFIG_SSL_CA_CERT), 'ssl_verify_host': 1,
            'ssl_verify_peer': 1, 'proxy_url': config.get(constants.CONFIG_PROXY_URL),
            'proxy_port': config.get(constants.CONFIG_PROXY_PORT),
            'proxy_user': config.get(constants.CONFIG_PROXY_USER),
            'proxy_password': config.get(constants.CONFIG_PROXY_PASSWORD)}
        downloader_config = DownloaderConfig(protocol='https', **downloader_config)

        # We will pass self as the event_listener, so that we can receive the callbacks in this class
        self.downloader = factory.get_downloader(downloader_config, self)
        self.progress_report = SyncProgressReport(sync_conduit)

    def process_relative_url(self, repo_id, importer_config,
                             distributor_config):
        """
        During create (but not update), if the relative path isn't specified it
        is derived from the feed_url.

        Ultimately, this belongs in the distributor itself. When we rewrite the
        yum distributor, we'll remove this entirely from the client.
        jdob, May 10, 2013
        """
        if constants.PUBLISH_RELATIVE_URL_KEYWORD not in distributor_config:
            if importer_constants.KEY_FEED in importer_config:
                if importer_config[importer_constants.KEY_FEED] is None:
                    self.prompt.render_failure_message(
                        _('Given repository feed URL is invalid.'))
                    return
                url_parse = urlparse(encode_unicode(
                    importer_config[importer_constants.KEY_FEED]))

                if url_parse[2] in ('', '/'):
                    relative_path = '/' + repo_id
                else:
                    relative_path = url_parse[2]
                distributor_config[constants.PUBLISH_RELATIVE_URL_KEYWORD] = relative_path  # noqa
            else:
                distributor_config[constants.PUBLISH_RELATIVE_URL_KEYWORD] = repo_id  # noqa

    def __init__(self, sync_conduit, config):
        """
        Initialize an ISOSyncRun.

        :param sync_conduit: the sync conduit to use for this sync run.
        :type  sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
        :param config:       plugin configuration
        :type  config:       pulp.plugins.config.PluginCallConfiguration
        """
        self.sync_conduit = sync_conduit
        self.config = config
        self._remove_missing_units = config.get(
            importer_constants.KEY_UNITS_REMOVE_MISSING,
            default=constants.CONFIG_UNITS_REMOVE_MISSING_DEFAULT)
        self._validate_downloads = config.get(importer_constants.KEY_VALIDATE,
                                              default=constants.CONFIG_VALIDATE_DEFAULT)
        self._repo_url = encode_unicode(config.get(importer_constants.KEY_FEED))
        # The _repo_url must end in a trailing slash, because we will use urljoin to determine
        # the path to
        # PULP_MANIFEST later
        if self._repo_url[-1] != '/':
            self._repo_url = self._repo_url + '/'

        # Cast our config parameters to the correct types and use them to build a Downloader
        max_speed = config.get(importer_constants.KEY_MAX_SPEED)
        if max_speed is not None:
            max_speed = float(max_speed)
        max_downloads = config.get(importer_constants.KEY_MAX_DOWNLOADS)
        if max_downloads is not None:
            max_downloads = int(max_downloads)
        else:
            max_downloads = constants.CONFIG_MAX_DOWNLOADS_DEFAULT
        ssl_validation = config.get_boolean(importer_constants.KEY_SSL_VALIDATION)
        ssl_validation = ssl_validation if ssl_validation is not None else \
            constants.CONFIG_VALIDATE_DEFAULT
        downloader_config = {
            'max_speed': max_speed,
            'max_concurrent': max_downloads,
            'ssl_client_cert': config.get(importer_constants.KEY_SSL_CLIENT_CERT),
            'ssl_client_key': config.get(importer_constants.KEY_SSL_CLIENT_KEY),
            'ssl_ca_cert': config.get(importer_constants.KEY_SSL_CA_CERT),
            'ssl_validation': ssl_validation,
            'proxy_url': config.get(importer_constants.KEY_PROXY_HOST),
            'proxy_port': config.get(importer_constants.KEY_PROXY_PORT),
            'proxy_username': config.get(importer_constants.KEY_PROXY_USER),
            'proxy_password': config.get(importer_constants.KEY_PROXY_PASS),
            'basic_auth_username': config.get(importer_constants.KEY_BASIC_AUTH_USER),
            'basic_auth_password': config.get(importer_constants.KEY_BASIC_AUTH_PASS),
            'working_dir': common_utils.get_working_directory()}
        downloader_config = DownloaderConfig(**downloader_config)

        # We will pass self as the event_listener, so that we can receive the callbacks in this
        # class
        if self._repo_url.lower().startswith('file'):
            self.downloader = LocalFileDownloader(downloader_config, self)
        else:
            self.downloader = HTTPThreadedDownloader(downloader_config, self)
        self.progress_report = SyncProgressReport(sync_conduit)

        self.repo_units = []

    def make_cert(self, cn, expiration, uid=None):
        """
        Generate an x509 certificate with the Subject set to the cn passed into this method:
        Subject: CN=someconsumer.example.com

        @param cn: ID to be embedded in the certificate
        @type  cn: string

        @param uid: The optional userid.  In pulp, this is the DB document _id
            for both users and consumers.
        @type uid: str

        @return: tuple of PEM encoded private key and certificate
        @rtype:  (str, str)
        """
        # Ensure we are dealing with a string and not unicode
        try:
            cn = str(cn)
        except UnicodeEncodeError:
            cn = encode_unicode(cn)

        log.debug("make_cert: [%s]" % cn)

        #Make a private key
        # Don't use M2Crypto directly as it leads to segfaults when trying to convert
        # the key to a PEM string.  Instead create the key with openssl and return the PEM string
        # Sorta hacky but necessary.
        # rsa = RSA.gen_key(1024, 65537, callback=passphrase_callback)
        private_key_pem = _make_priv_key()
        rsa = RSA.load_key_string(private_key_pem,
                                  callback=util.no_passphrase_callback)

        # Make the Cert Request
        req, pub_key = _make_cert_request(cn, rsa, uid=uid)

        # Sign it with the Pulp server CA
        # We can't do this in m2crypto either so we have to shell out

        ca_cert = config.config.get('security', 'cacert')
        ca_key = config.config.get('security', 'cakey')

        sn = SerialNumber()
        serial = sn.next()

        cmd = 'openssl x509 -req -sha1 -CA %s -CAkey %s -set_serial %s -days %d' % \
                (ca_cert, ca_key, serial, expiration)
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        output = p.communicate(input=req.as_pem())[0]
        p.wait()
        exit_code = p.returncode
        if exit_code != 0:
            raise Exception("error signing cert request: %s" % output)
        cert_pem_string = output[output.index("-----BEGIN CERTIFICATE-----"):]
        return private_key_pem, cert_pem_string

    def perform_sync(self, repo, sync_conduit, config):
        """
        Perform the sync operation accoring to the config for the given repo, and return a report.
        The sync progress will be reported through the sync_conduit.

        :param repo:         Metadata describing the repository
        :type  repo:         pulp.server.plugins.model.Repository
        :param sync_conduit: The sync_conduit that gives us access to the local repository
        :type  sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit
        :param config:       The configuration for the importer
        :type  config:       pulp.server.plugins.config.PluginCallConfiguration
        :return:             The sync report
        :rtype:              pulp.plugins.model.SyncReport
        """
        # Build the progress report and set it to the running state
        progress_report = SyncProgressReport(sync_conduit)

        # Cast our config parameters to the correct types and use them to build an ISOBumper
        max_speed = config.get(constants.CONFIG_MAX_SPEED)
        if max_speed is not None:
            max_speed = float(max_speed)
        num_threads = config.get(constants.CONFIG_NUM_THREADS)
        if num_threads is not None:
            num_threads = int(num_threads)
        else:
            num_threads = constants.DEFAULT_NUM_THREADS
        progress_report.metadata_state = STATE_RUNNING
        progress_report.update_progress()
        self.bumper = ISOBumper(
                           repo_url=encode_unicode(config.get(constants.CONFIG_FEED_URL)),
                           working_path=repo.working_dir,
                           max_speed=max_speed, num_threads=num_threads,
                           ssl_client_cert=config.get(constants.CONFIG_SSL_CLIENT_CERT),
                           ssl_client_key=config.get(constants.CONFIG_SSL_CLIENT_KEY),
                           ssl_ca_cert=config.get(constants.CONFIG_SSL_CA_CERT),
                           proxy_url=config.get(constants.CONFIG_PROXY_URL),
                           proxy_port=config.get(constants.CONFIG_PROXY_PORT),
                           proxy_user=config.get(constants.CONFIG_PROXY_USER),
                           proxy_password=config.get(constants.CONFIG_PROXY_PASSWORD))

        # Get the manifest and download the ISOs that we are missing
        manifest = self.bumper.get_manifest()
        progress_report.metadata_state = STATE_COMPLETE
        progress_report.modules_state = STATE_RUNNING
        progress_report.update_progress()
        missing_isos = self._filter_missing_isos(sync_conduit, manifest)
        new_isos = self.bumper.download_resources(missing_isos)

        # Move the downloaded stuff and junk to the permanent location
        self._create_units(sync_conduit, new_isos)

        # Report that we are finished
        progress_report.modules_state = STATE_COMPLETE
        report = progress_report.build_final_report()
        return report

    def __init__(self, id):
        """
        Creates a new instance, populating itself with the default values for all
        properties defined in PROPERTIES.

        @param id: unique identifier for the repo
        @type  id: string
        """
        self.id = encode_unicode(id)
        for k, d in self.PROPERTIES:
            self[k] = d

    def request(self, method, url, body):

        headers = dict(self.pulp_connection.headers)  # copy so we don't affect the calling method

        # Create a new connection each time since HTTPSConnection has problems
        # reusing a connection for multiple calls (lame).

        # Create SSL.Context and set it to verify peer SSL certificate against system CA certs.
        ssl_context = SSL.Context('sslv3')
        if self.pulp_connection.validate_ssl_ca:
            ssl_context.set_verify(SSL.verify_peer, 1)
            ssl_context.load_verify_locations(capath=self.pulp_connection.system_ca_dir)
        ssl_context.set_session_timeout(self.pulp_connection.timeout)

        if self.pulp_connection.username and self.pulp_connection.password:
            raw = ':'.join((self.pulp_connection.username, self.pulp_connection.password))
            encoded = base64.encodestring(raw)[:-1]
            headers['Authorization'] = 'Basic ' + encoded
        elif self.pulp_connection.cert_filename:
            ssl_context.load_cert(self.pulp_connection.cert_filename)

        # oauth configuration. This block is only True if oauth is not None, so it won't run on RHEL
        # 5.
        if self.pulp_connection.oauth_key and self.pulp_connection.oauth_secret and oauth:
            oauth_consumer = oauth.Consumer(
                self.pulp_connection.oauth_key,
                self.pulp_connection.oauth_secret)
            oauth_request = oauth.Request.from_consumer_and_token(
                oauth_consumer,
                http_method=method,
                http_url='https://%s:%d%s' % (self.pulp_connection.host, self.pulp_connection.port, url))
            oauth_request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), oauth_consumer, None)
            oauth_header = oauth_request.to_header()
            # unicode header values causes m2crypto to do odd things.
            for k, v in oauth_header.items():
                oauth_header[k] = encode_unicode(v)
            headers.update(oauth_header)
            headers['pulp-user'] = self.pulp_connection.oauth_user

        connection = httpslib.HTTPSConnection(
            self.pulp_connection.host, self.pulp_connection.port, ssl_context=ssl_context)

        try:
            # Request against the server
            connection.request(method, url, body=body, headers=headers)
            response = connection.getresponse()
        except SSL.SSLError, err:
            # Translate stale login certificate to an auth exception
            if 'sslv3 alert certificate expired' == str(err):
                raise exceptions.ClientSSLException(self.pulp_connection.cert_filename)
            else:
                raise exceptions.ConnectionException(None, str(err), None)

 def _ensure_input_encoding(self, input):
     """
     Recursively traverse any input structures and ensure any strings are
     encoded as utf-8
     @param input: input data
     @return: input data with strings encoded as utf-8
     """
     if isinstance(input, (list, set, tuple)):
         return [self._ensure_input_encoding(i) for i in input]
     if isinstance(input, dict):
         return dict((self._ensure_input_encoding(k), self._ensure_input_encoding(v)) for k, v in input.items())
     try:
         return encode_unicode(decode_unicode(input))
     except (UnicodeDecodeError, UnicodeEncodeError):
         raise InputEncodingError(input), None, sys.exc_info()[2]

 def _backup_existing_repodata(self):
     """
     Takes a backup of any existing repodata files. This is used in the final
     step where other file types in repomd.xml such as presto, updateinfo, comps
     are copied back to the repodata.
     """
     current_repo_dir = os.path.join(self.repodir, "repodata")
     # Note: backup_repo_dir is used to store presto metadata and possibly other custom metadata types
     # they will be copied back into new 'repodata' if needed.
     current_repo_dir = encode_unicode(current_repo_dir)
     if os.path.exists(current_repo_dir):
         _LOG.info("existing metadata found; taking backup.")
         self.backup_repodata_dir = os.path.join(self.repodir, "repodata.old")
         if os.path.exists(self.backup_repodata_dir):
             _LOG.debug("clean up any stale dirs")
             shutil.rmtree(self.backup_repodata_dir)
         shutil.copytree(current_repo_dir, self.backup_repodata_dir)
         os.system("chmod -R u+wX %s" % self.backup_repodata_dir)

def __get_groups_xml_info(repo_dir):
    groups_xml_path = None
    repodata_file = os.path.join(repo_dir, "repodata", "repomd.xml")
    repodata_file = encode_unicode(repodata_file)
    if os.path.isfile(repodata_file):
        ftypes = util.get_repomd_filetypes(repodata_file)
        _LOG.debug("repodata has filetypes of %s" % (ftypes))
        if "group" in ftypes:
            comps_ftype = util.get_repomd_filetype_path(
                    repodata_file, "group")
            filetype_path = os.path.join(repo_dir, comps_ftype)
            # createrepo uses filename as mdtype, rename to type.<ext>
            # to avoid filename too long errors
            renamed_filetype_path = os.path.join(os.path.dirname(comps_ftype),
                                     "comps" + '.' + '.'.join(os.path.basename(comps_ftype).split('.')[1:]))
            groups_xml_path = os.path.join(repo_dir, renamed_filetype_path)
            os.rename(filetype_path, groups_xml_path)
    return groups_xml_path

def _create_repo(dir, groups=None, checksum_type=DEFAULT_CHECKSUM):
    if not groups:
        cmd = "createrepo --database --checksum %s --update %s " % (checksum_type, dir)
    else:
        try:
            cmd = "createrepo --database --checksum %s -g %s --update %s " % (checksum_type, groups, dir)
        except UnicodeDecodeError:
            groups = decode_unicode(groups)
            cmd = "createrepo --database --checksum %s -g %s --update %s " % (checksum_type, groups, dir)
    # shlex now can handle unicode strings as well
    cmd = encode_unicode(cmd)
    try:
        cmd = shlex.split(cmd.encode('ascii', 'ignore'))
    except:
        cmd = shlex.split(cmd)

    _LOG.info("started repo metadata update: %s" % (cmd))
    handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return handle