Python utils.load_repos函数代码示例

def process(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    current_date = utils.date_to_timestamp('now')
    bug_modified_since = runtime_storage_inst.get_by_key('bug_modified_since')

    rcs_inst = rcs.get_rcs(cfg.CONF.review_uri)
    rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename,
                   username=cfg.CONF.ssh_username)

    for repo in repos:
        _process_repo(repo, runtime_storage_inst, record_processor_inst,
                      rcs_inst, bug_modified_since)

    rcs_inst.close()

    runtime_storage_inst.set_by_key('bug_modified_since', current_date)

    LOG.info('Processing mail lists')
    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
        _process_mail_list(mail_list, runtime_storage_inst,
                           record_processor_inst)

    _post_process_records(record_processor_inst, repos)

def process(runtime_storage_inst, default_data, sources_root, force_update):
    LOG.debug('Process default data')

    normalizer.normalize_default_data(default_data)

    dd_changed = _check_default_data_change(runtime_storage_inst, default_data)

    if 'project_sources' in default_data:
        if not _retrieve_project_list(default_data):
            raise Exception('Unable to retrieve project list')

    _update_default_data(runtime_storage_inst, default_data)

    if (dd_changed or force_update):
        LOG.debug('Gather release index for all repos')
        release_index = {}
        for repo in utils.load_repos(runtime_storage_inst):
            vcs_inst = vcs.get_vcs(repo, sources_root)
            release_index.update(vcs_inst.get_release_index())

        record_processor_inst = record_processor.RecordProcessor(
            runtime_storage_inst)
        # need to iterate over full view of records and generate valid
        # users profiles
        LOG.debug('Iterate all records to create valid users profiles')
        for record in runtime_storage_inst.get_all_records():
            record_processor_inst.update_user(record)
        # update records according to generated users profiles
        LOG.debug('Update all records according to users profiles')
        updated_records = record_processor_inst.update(
            runtime_storage_inst.get_all_records(), release_index)
        runtime_storage_inst.set_records(updated_records)

    def _get_modules(self):
        if self.modules is None:
            self.modules = set()
            self.alias_module_map = dict()

            for repo in utils.load_repos(self.runtime_storage_inst):
                module = repo['module'].lower()
                module_aliases = repo.get('aliases') or []

                add = True
                for module_name in ([module] + module_aliases):
                    for m in self.modules:
                        if module_name.find(m) >= 0:
                            add = False
                            break
                        if m.find(module_name) >= 0:
                            self.modules.remove(m)
                            break
                    if add:
                        self.modules.add(module_name)

                for alias in module_aliases:
                    self.alias_module_map[alias] = module

        return self.modules, self.alias_module_map

def _update_records(runtime_storage_inst, sources_root):
    LOG.debug("Update existing records")
    release_index = {}
    for repo in utils.load_repos(runtime_storage_inst):
        vcs_inst = vcs.get_vcs(repo, sources_root)
        release_index.update(vcs_inst.get_release_index())

    record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst)
    record_processor_inst.update(release_index)

def update_records(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    for repo in repos:
        process_repo(repo, runtime_storage_inst, record_processor_inst)

    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
        process_mail_list(mail_list, runtime_storage_inst,
                          record_processor_inst)

    record_processor_inst.update()

def process(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    for repo in repos:
        _process_repo(repo, runtime_storage_inst, record_processor_inst)

    LOG.info('Processing mail lists')
    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
        _process_mail_list(mail_list, runtime_storage_inst,
                           record_processor_inst)

    LOG.info('Processing translations stats')
    _process_translation_stats(runtime_storage_inst, record_processor_inst)

    _post_process_records(record_processor_inst, repos)

    def _get_modules(self):
        if self.modules is None:
            self.modules = set()
            for repo in utils.load_repos(self.runtime_storage_inst):
                module = repo['module'].lower()
                add = True
                for m in self.modules:
                    if module.find(m) >= 0:
                        add = False
                        break
                    if m.find(module) >= 0:
                        self.modules.remove(m)
                        break
                if add:
                    self.modules.add(module)

        return self.modules

def process(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    current_date = utils.date_to_timestamp('now')
    bug_modified_since = runtime_storage_inst.get_by_key('bug_modified_since')
    for repo in repos:
        _process_repo(repo, runtime_storage_inst, record_processor_inst,
                      bug_modified_since)
    runtime_storage_inst.set_by_key('bug_modified_since', current_date)

    LOG.info('Processing mail lists')
    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
        _process_mail_list(mail_list, runtime_storage_inst,
                           record_processor_inst)

    _post_process_records(record_processor_inst, repos)

def process(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    rcs_inst = rcs.get_rcs(cfg.CONF.review_uri)
    rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username)

    for repo in repos:
        _process_repo(repo, runtime_storage_inst, record_processor_inst, rcs_inst)

    rcs_inst.close()

    LOG.info("Processing mail lists")
    mail_lists = runtime_storage_inst.get_by_key("mail_lists") or []
    for mail_list in mail_lists:
        _process_mail_list(mail_list, runtime_storage_inst, record_processor_inst)

    _post_process_records(record_processor_inst, repos)

def _retrieve_project_list(runtime_storage_inst, project_sources):

    LOG.info('Retrieving project list from GitHub')

    repo_index = {}
    stored_repos = utils.load_repos(runtime_storage_inst)
    for repo in stored_repos:
        repo_index[repo['uri']] = repo

    github = MainClass.Github(timeout=60)

    for project_source in project_sources:
        organization = project_source['organization']
        LOG.debug('Get list of projects for organization %s', organization)
        try:
            repos = github.get_organization(organization).get_repos()
        except Exception as e:
            LOG.exception(e)
            LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
            return

        for repo in repos:
            repo_uri = repo.git_url
            repo_name = repo.name

            if repo_uri not in repo_index:
                r = {
                    'branches': ['master'],
                    'module': repo_name,
                    'organization': organization,
                    'project_type': project_source['project_type'],
                    'project_group': project_source['project_group'],
                    'uri': repo_uri,
                    'releases': []
                }
                stored_repos.append(r)
                LOG.debug('Project is added to default data: %s', r)

    runtime_storage_inst.set_by_key('repos', stored_repos)

def process(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

    rcs_inst = rcs.get_rcs(cfg.CONF.review_uri)
    rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename,
                   username=cfg.CONF.ssh_username)

    for repo in repos:
        _process_repo(repo, runtime_storage_inst, record_processor_inst,
                      rcs_inst)

    rcs_inst.close()

    LOG.info('Processing mail lists')
    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
        _process_mail_list(mail_list, runtime_storage_inst,
                           record_processor_inst)

    #TODO(adiantum): replace stub with tranlslation acquisition logic
    _process_translation(runtime_storage_inst, record_processor_inst)

    _post_process_records(record_processor_inst, repos)

def init_project_types(vault):
    runtime_storage_inst = vault['runtime_storage']
    project_type_options = {}
    project_type_group_index = {'all': set(['unknown'])}

    for repo in utils.load_repos(runtime_storage_inst):
        project_type = repo['project_type'].lower()
        project_group = None
        if ('project_group' in repo) and (repo['project_group']):
            project_group = repo['project_group'].lower()

        if project_type in project_type_options:
            if project_group:
                project_type_options[project_type].add(project_group)
        else:
            if project_group:
                project_type_options[project_type] = set([project_group])
            else:
                project_type_options[project_type] = set()

        module = repo['module']
        if project_type in project_type_group_index:
            project_type_group_index[project_type].add(module)
        else:
            project_type_group_index[project_type] = set([module])

        if project_group:
            if project_group in project_type_group_index:
                project_type_group_index[project_group].add(module)
            else:
                project_type_group_index[project_group] = set([module])

        project_type_group_index['all'].add(module)

    vault['project_type_options'] = project_type_options
    vault['project_type_group_index'] = project_type_group_index