Python syncLib.log函数代码示例

    def sync(self, channels=None):
        # If no channels specified, sync already synced channels
        if not channels:
            channels = self.synced_channels

        # Check channel availability before doing anything
        not_available = []
        for channel in channels:
            if any(channel not in d for d in
                   [self.channel_metadata, self.channel_to_family,  self.content_source_mapping]):
                not_available.append(channel)

        if not_available:
            raise ChannelNotFoundError("  " + "\n  ".join(not_available))

        # Need to update channel metadata
        self._update_channels_metadata(channels)

        # Finally, sync channel content
        total_time = datetime.timedelta()
        for channel in channels:
            cur_time = self._sync_channel(channel)
            total_time += cur_time
            # Switch back to cdnsync log
            rhnLog.initLOG(self.log_path, self.log_level)
            log2disk(0, "Sync of channel completed.")

        log(0, "Total time: %s" % str(total_time).split('.')[0])

    def import_channel_families(self):
        """Insert channel family data into DB."""

        log(1, "Channel families in manifest: %d" % len(self.sat5_cert.channel_families))  # pylint: disable=E1101

        batch = []
        for cf in self.sat5_cert.channel_families:  # pylint: disable=E1101
            label = cf.name
            try:
                family = self.families[label]
                family_object = ChannelFamily()
                for k in family.keys():
                    family_object[k] = family[k]
                family_object['label'] = label
                batch.append(family_object)
                self.families_to_import.append(label)
            except KeyError:
                # While channel mappings are not consistent with certificate generated on RHN...
                msg = ("WARNING: Channel family '%s' is provided by manifest but "
                       "was not found in cdn-sync mappings." % label)
                log2(0, 1, msg, stream=sys.stderr)

        log(1, "Channel families to import: %d" % len(batch))
        # Perform import
        backend = SQLBackend()
        importer = ChannelFamilyImport(batch, backend)
        importer.run()

    def _list_available_channels(self):
        # Select from rhnContentSsl to filter cdn-activated channel families
        h = rhnSQL.prepare("""
            select label from rhnChannelFamilyPermissions cfp inner join
                              rhnChannelFamily cf on cfp.channel_family_id = cf.id inner join
                              rhnContentSsl cs on cf.id = cs.channel_family_id
            where cf.org_id is null
        """)
        h.execute()
        families = h.fetchall_dict() or []

        # collect all channel from available families
        all_channels = []
        base_channels = {}
        for family in families:
            label = family['label']
            family = self.families[label]
            channels = [c for c in family['channels'] if c is not None]
            all_channels.extend(channels)

        # fill base_channel
        for channel in all_channels:
            try:
                # Only base channels as key in dictionary
                if self.channel_metadata[channel]['parent_channel'] is None:
                    base_channels[channel] = [k for k in all_channels
                                              if self.channel_metadata[k]['parent_channel'] == channel]
            except KeyError:
                log(1, "Channel %s not found in channel metadata" % channel)
                continue

        return base_channels

    def __init__(self, local_mount_point=None, client_cert_id=None):
        rhnSQL.initDB()
        self.local_mount_point = local_mount_point
        self.repository_tree = CdnRepositoryTree()
        self._populate_repository_tree(client_cert_id=client_cert_id)

        f = None
        try:
            try:
                # Channel to repositories mapping
                f = open(constants.CONTENT_SOURCE_MAPPING_PATH, 'r')
                self.content_source_mapping = json.load(f)
                f.close()

                # Channel to kickstart repositories mapping
                f = open(constants.KICKSTART_SOURCE_MAPPING_PATH, 'r')
                self.kickstart_source_mapping = json.load(f)
                f.close()

                # Kickstart metadata
                f = open(constants.KICKSTART_DEFINITIONS_PATH, 'r')
                self.kickstart_metadata = json.load(f)
                f.close()
            except IOError:
                e = sys.exc_info()[1]
                log(1, "Ignoring channel mappings: %s" % e)
                self.content_source_mapping = {}
                self.kickstart_source_mapping = {}
                self.kickstart_metadata = {}
        finally:
            if f is not None:
                f.close()

        self.__init_repository_to_channels_mapping()

    def sync(self, channels=None):
        # If no channels specified, sync already synced channels
        if not channels:
            channels = self.synced_channels

        # Check channel availability before doing anything
        not_available = []
        for channel in channels:
            if any(channel not in d for d in
                   [self.channel_metadata, self.channel_to_family]) or (
                       not self.cdn_repository_manager.check_channel_availability(channel, self.no_kickstarts)):
                not_available.append(channel)

        if not_available:
            raise ChannelNotFoundError("  " + "\n  ".join(not_available))

        # Need to update channel metadata
        self._update_channels_metadata(channels)

        # Finally, sync channel content
        error_messages = []
        total_time = datetime.timedelta()
        for channel in channels:
            cur_time, ret_code = self._sync_channel(channel)
            if ret_code != 0:
                error_messages.append("Problems occurred during syncing channel %s. Please check "
                                      "/var/log/rhn/cdnsync/%s.log for the details\n" % (channel, channel))
            total_time += cur_time
            # Switch back to cdnsync log
            rhnLog.initLOG(self.log_path, self.log_level)
            log2disk(0, "Sync of channel completed.")
        log(0, "Total time: %s" % str(total_time).split('.')[0])
        return error_messages

    def _get_content_sources(self, channel, backend):
        batch = []
        sources = []
        type_id = backend.lookupContentSourceType('yum')

        if channel in self.content_source_mapping:
            sources.extend(self.content_source_mapping[channel])
        for source in sources:
            if not source['pulp_content_category'] == "source":
                content_source = ContentSource()
                content_source['label'] = source['pulp_repo_label_v2']
                content_source['source_url'] = CFG.CDN_ROOT + source['relative_url']
                content_source['org_id'] = None
                content_source['type_id'] = type_id
                batch.append(content_source)

        if channel in self.kickstart_metadata:
            for tree in self.kickstart_metadata[channel]:
                tree_label = tree['ks_tree_label']
                if tree_label in self.kickstart_source_mapping:
                    sources = self.kickstart_source_mapping[tree_label]
                    # One tree comes from one repo, one repo for each tree is in the mapping,
                    # in future there may be multiple repos for one tree and we will need to select
                    # correct repo
                    source = sources[0]
                    content_source = ContentSource()
                    content_source['label'] = tree_label
                    content_source['source_url'] = CFG.CDN_ROOT + source['relative_url']
                    content_source['org_id'] = None
                    content_source['type_id'] = type_id
                    batch.append(content_source)
                else:
                    log(1, "WARN: Kickstart tree not available: %s" % tree_label)

        return batch

 def _print_unmapped_channels(self):
     unmapped_channels = [ch for ch in self.synced_channels if not self.synced_channels[ch]
                          and ch not in self.channel_metadata]
     if unmapped_channels:
         log(0, "Previously synced channels not available to update from CDN:")
         for channel in sorted(unmapped_channels):
             log(0, "    p %s" % channel)

    def __init__(self, current_manifest=None, username=None, password=None,
                 http_proxy=None, http_proxy_username=None, http_proxy_password=None):
        self.base_url = current_manifest.get_api_url()
        if CFG.CANDLEPIN_SERVER_API:
            log(0, "Overriding Candlepin server to: '%s'" % CFG.CANDLEPIN_SERVER_API)
            self.base_url = CFG.CANDLEPIN_SERVER_API

        if self.base_url.startswith('https'):
            self.protocol = 'https'
        elif self.base_url.startswith('http'):
            self.protocol = 'http'
        else:
            raise ValueError("Invalid protocol in URL: '%s'" % self.base_url)

        if not self.base_url.endswith('/'):
            self.base_url += '/'

        self.current_manifest = current_manifest

        # Authentication with upstream consumer certificate or with username and password
        if self.current_manifest and self.protocol == 'https' and not username:
            self.username = self.password = None
        else:
            log(0, "Candlepin login:")
            self.username, self.password = getUsernamePassword(username, password)

        self.http_proxy = http_proxy
        self.http_proxy_username = http_proxy_username
        self.http_proxy_password = http_proxy_password

    def get_crypto_keys(self, check_dates=False):
        ssl_query = rhnSQL.prepare("""
            select description, key, org_id from rhnCryptoKey where id = :id
        """)
        keys = {}
        ssl_query.execute(id=self.ca_cert)
        row = ssl_query.fetchone_dict()
        keys['ca_cert'] = (str(row['description']), str(row['key']), row['org_id'])
        ssl_query.execute(id=self.client_cert)
        row = ssl_query.fetchone_dict()
        keys['client_cert'] = (str(row['description']), str(row['key']), row['org_id'])
        ssl_query.execute(id=self.client_key)
        row = ssl_query.fetchone_dict()
        keys['client_key'] = (str(row['description']), str(row['key']), row['org_id'])

        # Check if SSL certificates are usable
        if check_dates:
            failed = 0
            for key in (keys['ca_cert'], keys['client_cert']):
                if not verify_certificate_dates(key[1]):
                    log(1, "WARNING: Problem with dates in certificate '%s'. "
                           "Please check validity of this certificate." % key[0])
                    failed += 1
            if failed:
                return {}
        return keys

 def deactivate():
     """Function to remove certificates and manifest repositories from DB"""
     rhnSQL.initDB()
     log(0, "Removing certificates...")
     Activation._remove_certificates()
     log(0, "Removing manifest repositories...")
     Activation._remove_repositories()

def enableSatelliteRepo(rhn_cert):
    args = ['rpm', '-q', '--qf', '\'%{version} %{arch}\'', '-f', '/etc/redhat-release']
    ret, out, err = fileutils.rhn_popen(args)
    data = out.read().strip("'")
    version, arch = data.split()
    # Read from stdout, strip quotes if any and extract first number
    version = re.search(r'\d+', version).group()

    if version not in SUPPORTED_RHEL_VERSIONS:
        log(0, "WARNING: No Satellite repository available for RHEL version: %s." % version)
        return

    arch_str = "server"
    if arch == "s390x":
        arch_str = "system-z"

    sat_cert = satellite_cert.SatelliteCert()
    sat_cert.load(rhn_cert)
    sat_version = getattr(sat_cert, 'satellite-version')

    repo = "rhel-%s-%s-satellite-%s-rpms" % (version, arch_str, sat_version)
    args = ['/usr/bin/subscription-manager', 'repos', '--enable', repo]
    ret, out, err = fileutils.rhn_popen(args)
    if ret:
        msg_ = "Enabling of Satellite repository failed."
        msg = ("%s\nReturn value: %s\nStandard-out: %s\n\n"
               "Standard-error: %s\n"
               % (msg_, ret, out.read(), err.read()))
        writeError(msg)
        raise EnableSatelliteRepositoryException("Enabling of Satellite repository failed. Make sure Satellite "
                                                 "subscription is attached to this system, both versions of RHEL and "
                                                 "Satellite are supported or run activation with --disconnected "
                                                 "option.")

    def setup_repos_and_sync(self, channels=None, add_repos=None, delete_repos=None):
        # Fix format of relative url
        if add_repos:
            for index, repo in enumerate(add_repos):
                repo = repo.replace(CFG.CDN_ROOT, '')
                repo = os.path.join('/', repo)
                add_repos[index] = repo
        if delete_repos:
            for index, repo in enumerate(delete_repos):
                repo = repo.replace(CFG.CDN_ROOT, '')
                repo = os.path.join('/', repo)
                delete_repos[index] = repo
        # We need single custom channel
        if not channels or len(channels) > 1:
            raise CustomChannelSyncError("Single custom channel needed.")
        channel = list(channels)[0]
        db_channel = channel_info(channel)
        if add_repos and not self._can_add_repos(db_channel, add_repos):
            raise CustomChannelSyncError("Unable to attach requested repositories to this channel.")
        # Add custom repositories to custom channel
        new_repos_count = self.cdn_repository_manager.assign_repositories_to_channel(channel, delete_repos=delete_repos,
                                                                                     add_repos=add_repos)
        if new_repos_count:
            # Add to synced channels if there are any repos
            if channel not in self.synced_channels:
                self.synced_channels[channel] = db_channel['org_id']
            error_messages = self.sync(channels=channels)
        else:
            log(0, "No repositories attached to channel. Skipping sync.")
            error_messages = None

        return error_messages

    def assign_repositories_to_channel(self, channel_label, delete_repos=None, add_repos=None):
        backend = SQLBackend()
        self.unlink_all_repos(channel_label, custom_only=True)
        repos = self.list_associated_repos(channel_label)
        changed = 0
        if delete_repos:
            for to_delete in delete_repos:
                if to_delete in repos:
                    repos.remove(to_delete)
                    log(0, "Removing repository '%s' from channel." % to_delete)
                    changed += 1
                else:
                    log2(0, 0, "WARNING: Repository '%s' is not attached to channel." % to_delete, stream=sys.stderr)
        if add_repos:
            for to_add in add_repos:
                if to_add not in repos:
                    repos.append(to_add)
                    log(0, "Attaching repository '%s' to channel." % to_add)
                    changed += 1
                else:
                    log2(0, 0, "WARNING: Repository '%s' is already attached to channel." % to_add, stream=sys.stderr)

        # If there are any repositories intended to be attached to channel
        if repos:
            content_sources_batch = self.get_content_sources_import_batch(
                channel_label, backend, repos=sorted(repos))
            for content_source in content_sources_batch:
                content_source['channels'] = [channel_label]
                importer = ContentSourcesImport(content_sources_batch, backend)
                importer.run()
        else:
            # Make sure everything is unlinked
            self.unlink_all_repos(channel_label)
        return changed

    def run(self):
        size = 0
        for queue in self.queues.values():
            size += queue.qsize()
        if size <= 0:
            return
        log(1, "Downloading total %d files from %d queues." % (size, len(self.queues)))

        for index, queue in enumerate(self.queues.values()):
            log(2, "Downloading %d files from queue #%d." % (queue.qsize(), index))
            self.first_in_queue_done = False
            started_threads = []
            for _ in range(self.threads):
                thread = DownloadThread(self, queue)
                thread.setDaemon(True)
                thread.start()
                started_threads.append(thread)

            # wait to finish
            try:
                while any(t.isAlive() for t in started_threads):
                    time.sleep(1)
            except KeyboardInterrupt:
                e = sys.exc_info()[1]
                self.fail_download(e)
                while any(t.isAlive() for t in started_threads):
                    time.sleep(1)
                break

        # raise first detected exception if any
        if self.exception:
            raise self.exception  # pylint: disable=E0702

 def log(self, *_):
     self.lock.acquire()
     self.status += 1
     self._print_progress_bar(self.status, self.total, prefix=self.msg, bar_length=50)
     if time.time() > int(self.last_log + 90):
         self.last_log = time.time()
         log(0, '%s %s' % (round(100.00 * (self.status / float(self.total)), 2), '%'))
     self.lock.release()