Python stagingapi.StagingAPI类代码示例

class CheckRepo(object):

    def __init__(self, apiurl, project, readonly=False, force_clean=False, debug=False):
        """CheckRepo constructor."""
        self.apiurl = apiurl
        self.project = project
        self.staging = StagingAPI(apiurl, self.project)

        self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)

        # grouped = { id: staging, }
        self.grouped = {}
        # groups = { staging: [ids,], }
        self.groups = {}
        self._staging()
        self.readonly = readonly
        self.debug_enable = debug

    def debug(self, *args):
        if not self.debug_enable:
            return
        print ' '.join([i if isinstance(i, basestring) else pformat(i) for i in args])

    def _staging(self):
        """Preload the groups of related request associated by the same
        staging project.

        """
        for project in self.staging.get_staging_projects():
            # Get all the requests identifier for the project
            requests = self.staging.get_prj_pseudometa(project)['requests']
            requests = [req['id'] for req in requests]

            # Note: Originally we recover also the request returned by
            # list_requests_in_prj().  I guest that if the staging
            # project is working properly, this method do not add any
            # new request to the list.
            if requests:
                self.groups[project] = requests
                self.grouped.update({req: project for req in requests})

    def get_request_state(self, request_id):
        """Return the current state of the request."""
        state = None
        url = makeurl(self.apiurl, ('request', str(request_id)))
        try:
            root = ET.parse(http_GET(url)).getroot()
            state = root.find('state').get('name')
        except urllib2.HTTPError, e:
            print('ERROR in URL %s [%s]' % (url, e))
        return state

 def setUp(self):
     """
     Initialize the configuration
     """
     self.obs = obs.OBS()
     Config(obs.APIURL, 'openSUSE:Factory')
     self.api = StagingAPI(obs.APIURL, 'openSUSE:Factory')

    def setUp(self):
        """
        Initialize the configuration
        """

        self.obs = OBS()
        self.api = StagingAPI(APIURL)

    def __init__(self, project=PROJECT):
        """
        Initialize the configuration
        """
        THIS_DIR = os.path.dirname(os.path.abspath(__file__))
        oscrc = os.path.join(THIS_DIR, 'test.oscrc')

        self.apiurl = APIURL
        logging.basicConfig()

        # clear cache from other tests - otherwise the VCR is replayed depending
        # on test order, which can be harmful
        memoize_session_reset()

        osc.core.conf.get_config(override_conffile=oscrc,
                                 override_no_keyring=True,
                                 override_no_gnome_keyring=True)
        if os.environ.get('OSC_DEBUG'):
            osc.core.conf.config['debug'] = 1
        self.project = project
        self.projects = {}
        self.requests = []
        self.groups = []
        self.users = []
        CacheManager.test = True
        # disable caching, the TTLs break any reproduciblity
        Cache.CACHE_DIR = None
        Cache.PATTERNS = {}
        Cache.init()
        self.setup_remote_config()
        self.load_config()
        self.api = StagingAPI(APIURL, project)

 def __init__(self, name):
     self.name = name
     Config(apiurl, name)
     self.api = StagingAPI(apiurl, name)
     self.staging_projects = dict()
     self.listener = None
     self.logger = logging.getLogger(__name__)
     self.replace_string = self.api.attribute_value_load('OpenQAMapping')

    def setUp(self):
        """
        Initialize the configuration
        """

        self.obs = OBS()
        Config("openSUSE:Factory")
        self.api = StagingAPI(APIURL, "openSUSE:Factory")

class TestUnselect(unittest.TestCase):
    def setUp(self):
        self.obs = obs.OBS()
        Config(obs.APIURL, obs.PROJECT)
        self.api = StagingAPI(obs.APIURL, obs.PROJECT)

    def test_cleanup_filter(self):
        UnselectCommand.config_init(self.api)
        UnselectCommand.cleanup_days = 1
        obsolete = self.api.project_status_requests('obsolete', UnselectCommand.filter_obsolete)
        self.assertSequenceEqual(['627445', '642126', '646560', '645723', '646823'], obsolete)

class TestSelect(unittest.TestCase):

    def setUp(self):
        """
        Initialize the configuration
        """
        self.obs = obs.OBS()
        Config(obs.APIURL, 'openSUSE:Factory')
        self.api = StagingAPI(obs.APIURL, 'openSUSE:Factory')

    def test_old_frozen(self):
        self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False)
        # check it won't allow selecting
        self.assertEqual(False, SelectCommand(self.api, 'openSUSE:Factory:Staging:A').perform(['gcc']))

    def test_select_comments(self):
        c_api = CommentAPI(self.api.apiurl)
        staging_b = 'openSUSE:Factory:Staging:B'
        comments = c_api.get_comments(project_name=staging_b)

        # First select
        self.assertEqual(True, SelectCommand(self.api, staging_b).perform(['gcc', 'wine']))
        first_select_comments = c_api.get_comments(project_name=staging_b)
        last_id = sorted(first_select_comments.keys())[-1]
        first_select_comment = first_select_comments[last_id]
        # Only one comment is added
        self.assertEqual(len(first_select_comments), len(comments) + 1)
        # With the right content
        self.assertTrue('request#123 for package gcc submitted by Admin' in first_select_comment['comment'])

        # Second select
        self.assertEqual(True, SelectCommand(self.api, staging_b).perform(['puppet']))
        second_select_comments = c_api.get_comments(project_name=staging_b)
        last_id = sorted(second_select_comments.keys())[-1]
        second_select_comment = second_select_comments[last_id]
        # The number of comments increased by one
        self.assertEqual(len(second_select_comments) - 1, len(first_select_comments))
        self.assertNotEqual(second_select_comment['comment'], first_select_comment['comment'])
        # The new comments contains new, but not old
        self.assertFalse('request#123 for package gcc submitted by Admin' in second_select_comment['comment'])
        self.assertTrue('added request#321 for package puppet submitted by Admin' in second_select_comment['comment'])

    def test_no_matches(self):
        # search for requests
        with self.assertRaises(oscerr.WrongArgs) as cm:
            SelectCommand(self.api, 'openSUSE:Factory:Staging:B').perform(['bash'])
        self.assertEqual(str(cm.exception), "No SR# found for: bash")

    def test_selected(self):
        # make sure the project is frozen recently for other tests

        ret = SelectCommand(self.api, 'openSUSE:Factory:Staging:B').perform(['wine'])
        self.assertEqual(True, ret)

    def __init__(self, apiurl, project, readonly=False, force_clean=False, debug=False):
        """CheckRepo constructor."""
        self.apiurl = apiurl
        self.project = project
        self.staging = StagingAPI(apiurl, self.project)

        self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)

        # grouped = { id: staging, }
        self.grouped = {}
        # groups = { staging: [ids,], }
        self.groups = {}
        self._staging()
        self.readonly = readonly
        self.debug_enable = debug

def do_staging(self, subcmd, opts, *args):
    """${cmd_name}: Commands to work with staging projects

    ${cmd_option_list}

    "accept" will accept all requests in
        openSUSE:Factory:Staging:<LETTER> (into Factory)

    "acheck" will check if it's safe to accept new staging projects
        As openSUSE:Factory is syncing the right package versions between
        /standard, /totest and /snapshot, it's important that the projects
        are clean prior to a checkin round.

    "check" will check if all packages are links without changes

    "cleanup_rings" will try to cleanup rings content and print
        out problems

    "freeze" will freeze the sources of the project's links (not
        affecting the packages actually in)

    "frozenage" will show when the respective staging project was last frozen

    "list" will pick the requests not in rings

    "select" will add requests to the project

    "unselect" will remove from the project - pushing them back to the backlog

    Usage:
        osc staging accept [--force] [LETTER...]
        osc staging check [--old] REPO
        osc staging cleanup_rings
        osc staging freeze [--no-boostrap] PROJECT...
        osc staging frozenage PROJECT...
        osc staging list [--supersede]
        osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST...
        osc staging unselect REQUEST...
        osc staging repair REQUEST...
    """
    if opts.version:
        self._print_version()

    # verify the argument counts match the commands
    if len(args) == 0:
        raise oscerr.WrongArgs('No command given, see "osc help staging"!')
    cmd = args[0]
    if cmd in ("freeze", "frozenage", "repair"):
        min_args, max_args = 1, None
    elif cmd == "check":
        min_args, max_args = 0, 2
    elif cmd == "select":
        min_args, max_args = 1, None
        if not opts.add:
            min_args = 2
    elif cmd == "unselect":
        min_args, max_args = 1, None
    elif cmd == "adi":
        min_args, max_args = None, None
    elif cmd in ("list", "accept"):
        min_args, max_args = 0, None
    elif cmd in ("cleanup_rings", "acheck"):
        min_args, max_args = 0, 0
    else:
        raise oscerr.WrongArgs("Unknown command: %s" % cmd)
    if len(args) - 1 < min_args:
        raise oscerr.WrongArgs("Too few arguments.")
    if max_args is not None and len(args) - 1 > max_args:
        raise oscerr.WrongArgs("Too many arguments.")

    # Init the OBS access and configuration
    opts.project = self._full_project_name(opts.project)
    opts.apiurl = self.get_api_url()
    opts.verbose = False
    Config(opts.project)

    with OBSLock(opts.apiurl, opts.project):
        api = StagingAPI(opts.apiurl, opts.project)

        # call the respective command and parse args by need
        if cmd == "check":
            prj = args[1] if len(args) > 1 else None
            CheckCommand(api).perform(prj, opts.old)
        elif cmd == "freeze":
            for prj in args[1:]:
                FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap=opts.bootstrap)
        elif cmd == "frozenage":
            for prj in args[1:]:
                print "%s last frozen %0.1f days ago" % (
                    api.prj_from_letter(prj),
                    api.days_since_last_freeze(api.prj_from_letter(prj)),
                )
        elif cmd == "acheck":
            # Is it safe to accept? Meaning: /totest contains what it should and is not dirty
            version_totest = api.get_binary_version(
                api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64"
            )
            if version_totest:
                version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest")
                totest_dirty = api.is_repo_dirty(api.project, "totest")
#.........这里部分代码省略.........

 def __init__(self, project, dryrun = False):
     self.project = project
     self.dryrun = dryrun
     self.api = StagingAPI(osc.conf.config['apiurl'], project='openSUSE:%s' % project)
     self.known_failures = self.known_failures_from_dashboard(project)

class Project(object):
    def __init__(self, name):
        self.name = name
        Config(apiurl, name)
        self.api = StagingAPI(apiurl, name)
        self.staging_projects = dict()
        self.listener = None
        self.logger = logging.getLogger(__name__)
        self.replace_string = self.api.attribute_value_load('OpenQAMapping')

    def init(self):
        for p in self.api.get_staging_projects():
            if self.api.is_adi_project(p):
                continue
            self.staging_projects[p] = self.initial_staging_state(p)
            self.update_staging_status(p)

    def staging_letter(self, name):
        return name.split(':')[-1]

    def map_iso(self, staging_project, iso):
        parts = self.replace_string.split('/')
        if parts[0] != 's':
            raise Exception("{}'s iso_replace_string does not start with s/".format(self.name))
        old = parts[1]
        new = parts[2]
        new = new.replace('$LETTER', self.staging_letter(staging_project))
        return re.compile(old).sub(new, iso)

    def gather_isos(self, name, repository):
        url = self.api.makeurl(['published', name, repository, 'iso'])
        f = self.api.retried_GET(url)
        root = ET.parse(f).getroot()
        ret = []
        for entry in root.findall('entry'):
            if entry.get('name').endswith('iso'):
                ret.append(self.map_iso(name, entry.get('name')))
        return ret

    def gather_buildid(self, name, repository):
        url = self.api.makeurl(['published', name, repository], {'view': 'status'})
        f = self.api.retried_GET(url)
        id = ET.parse(f).getroot().find('buildid')
        if id is not None:
            return id.text

    def initial_staging_state(self, name):
        return {'isos': self.gather_isos(name, 'images'),
                'id': self.gather_buildid(name, 'images')}

    def fetch_openqa_jobs(self, staging, iso):
        buildid = self.staging_projects[staging].get('id')
        if not buildid:
            self.logger.info("I don't know the build id of " + staging)
            return
        # all openQA jobs are created at the same URL
        url = self.api.makeurl(['status_reports', 'published', staging, 'images', 'reports', buildid])
        openqa = self.listener.jobs_for_iso(iso)
        # collect job infos to pick names
        openqa_infos = dict()
        for job in openqa:
            print(staging, iso, job['id'], job['state'], job['result'],
                  job['settings']['MACHINE'], job['settings']['TEST'])
            openqa_infos[job['id']] = {'url': self.listener.test_url(job)}
            openqa_infos[job['id']]['state'] = self.map_openqa_result(job)
            openqa_infos[job['id']]['name'] = job['settings']['TEST']
            openqa_infos[job['id']]['machine'] = job['settings']['MACHINE']

        # make sure the names are unique
        taken_names = dict()
        for id in openqa_infos:
            name = openqa_infos[id]['name']
            if name in taken_names:
                openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" + openqa_infos[id]['machine']
                # the other id
                id = taken_names[name]
                openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" + openqa_infos[id]['machine']
            taken_names[name] = id

        for info in openqa_infos.values():
            xml = self.openqa_check_xml(info['url'], info['state'], 'openqa:' + info['name'])
            try:
                http_POST(url, data=xml)
            except HTTPError:
                self.logger.error('failed to post status to ' + url)

    def update_staging_status(self, project):
        for iso in self.staging_projects[project]['isos']:
            self.fetch_openqa_jobs(project, iso)

    def update_staging_buildid(self, project, repository, buildid):
        self.staging_projects[project]['id'] = buildid
        self.staging_projects[project]['isos'] = self.gather_isos(project, repository)
        self.update_staging_status(project)

    def check_published_repo(self, project, repository, buildid):
        if repository != 'images':
            return
        for p in self.staging_projects:
            if project == p:
#.........这里部分代码省略.........

class TestApiCalls(unittest.TestCase):
    """
    Tests for various api calls to ensure we return expected content
    """

    def setUp(self):
        """
        Initialize the configuration
        """

        self.obs = OBS()
        Config('openSUSE:Factory')
        self.api = StagingAPI(APIURL, 'openSUSE:Factory')

    def tearDown(self):
        """Clean internal cache"""
        if hasattr(self.api, '_invalidate_all'):
            self.api._invalidate_all()

    def test_ring_packages(self):
        """
        Validate the creation of the rings.
        """
        # our content in the XML files
        # test content for listonly ie. list command
        ring_packages = {
            'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
            'elem-ring-1': 'openSUSE:Factory:Rings:0-Bootstrap',
            'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
            'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD',
            'git': 'openSUSE:Factory:Rings:2-TestDVD',
            'wine': 'openSUSE:Factory:Rings:1-MinimalX',
        }
        self.assertEqual(ring_packages, self.api.ring_packages_for_links)

        # test content for real usage
        ring_packages = {
            'elem-ring-0': 'openSUSE:Factory:Rings:0-Bootstrap',
            'elem-ring-1': 'openSUSE:Factory:Rings:1-MinimalX',
            'elem-ring-mini': 'openSUSE:Factory:Rings:0-Bootstrap',
            'elem-ring-2': 'openSUSE:Factory:Rings:2-TestDVD',
            'git': 'openSUSE:Factory:Rings:2-TestDVD',
            'wine': 'openSUSE:Factory:Rings:1-MinimalX',
        }
        self.assertEqual(ring_packages, self.api.ring_packages)

    @unittest.skip("no longer approving non-ring packages")
    def test_dispatch_open_requests(self):
        """
        Test dispatching and closure of non-ring packages
        """

        # Get rid of open requests
        self.api.dispatch_open_requests()
        # Check that we tried to close it
        self.assertEqual(httpretty.last_request().method, 'POST')
        self.assertEqual(httpretty.last_request().querystring[u'cmd'], [u'changereviewstate'])
        # Try it again
        self.api.dispatch_open_requests()
        # This time there should be nothing to close
        self.assertEqual(httpretty.last_request().method, 'GET')

    def test_pseudometa_get_prj(self):
        """
        Test getting project metadata from YAML in project description
        """

        # Try to get data from project that has no metadata
        data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
        # Should be empty, but contain structure to work with
        self.assertEqual(data, {'requests': []})
        # Add some sample data
        rq = {'id': '123', 'package': 'test-package'}
        data['requests'].append(rq)
        # Save them and read them back
        self.api.set_prj_pseudometa('openSUSE:Factory:Staging:A', data)
        test_data = self.api.get_prj_pseudometa('openSUSE:Factory:Staging:A')
        # Verify that we got back the same data
        self.assertEqual(data, test_data)

    def test_list_projects(self):
        """
        List projects and their content
        """

        # Prepare expected results
        data = []
        for prj in self.obs.staging_project:
            data.append('openSUSE:Factory:Staging:' + prj)

        # Compare the results
        self.assertEqual(data, self.api.get_staging_projects())

    def test_open_requests(self):
        """
        Test searching for open requests
        """

        requests = []

#.........这里部分代码省略.........

class ToTestBase(object):
    """Base class to store the basic interface"""

    def __init__(self, project, dryrun = False):
        self.project = project
        self.dryrun = dryrun
        self.api = StagingAPI(osc.conf.config['apiurl'], project='openSUSE:%s' % project)
        self.known_failures = self.known_failures_from_dashboard(project)

    def openqa_group(self):
        return self.project

    def iso_prefix(self):
        return self.project

    def jobs_num(self):
        return 70

    def current_version(self):
        return self.release_version()

    def binaries_of_product(self, project, product):
        url = self.api.makeurl(['build', project, 'images', 'local', product])
        try:
            f = self.api.retried_GET(url)
        except urllib2.HTTPError:
            return []

        ret = []
        root = ET.parse(f).getroot()
        for binary in root.findall('binary'):
            ret.append(binary.get('filename'))

        return ret

    def get_current_snapshot(self):
        """Return the current snapshot in :ToTest"""

        # for now we hardcode all kind of things
        for binary in self.binaries_of_product('openSUSE:%s:ToTest' % self.project, '_product:openSUSE-cd-mini-%s' % self.arch()):
            result = re.match(r'openSUSE-%s-NET-.*-Snapshot(.*)-Media.iso' % self.iso_prefix(),
                              binary)
            if result:
                return result.group(1)

        return None


    def ftp_build_version(self, tree):
        for binary in self.binaries_of_product('openSUSE:%s' % self.project, tree):
            result = re.match(r'openSUSE.*Build(.*)-Media1.report', binary)

        if result:
            return result.group(1)
        raise Exception("can't find %s version" % self.project)

    def release_version(self):
       url = self.api.makeurl(['build', 'openSUSE:%s' % self.project, 'standard', self.arch(),
                               '_product:openSUSE-release'])
       f = self.api.retried_GET(url)
       root = ET.parse(f).getroot()
       for binary in root.findall('binary'):
           binary = binary.get('filename', '')
           result = re.match(r'.*-([^-]*)-[^-]*.src.rpm', binary)
           if result:
              return result.group(1)

       raise Exception("can't find %s version" % self.project)

    def find_openqa_results(self, snapshot):
        """Return the openqa jobs of a given snapshot and filter out the
        cloned jobs

        """

        url = makeurl('https://openqa.opensuse.org', ['api', 'v1', 'jobs'], { 'group': self.openqa_group(), 'build': snapshot } )
        f = self.api.retried_GET(url)
        jobs = []
        for job in json.load(f)['jobs']:
            if job['clone_id'] or job['result'] == 'obsoleted':
                continue
            job['name'] = job['name'].replace(snapshot, '')
            jobs.append(job)
        return jobs

    def _result2str(self, result):
        if result == QA_INPROGRESS:
            return 'inprogress'
        elif result == QA_FAILED:
            return 'failed'
        else:
            return 'passed'

    def find_failed_module(self, testmodules):
        # print json.dumps(testmodules, sort_keys=True, indent=4)
        for module in testmodules:
            if module['result'] != 'failed':
                continue
            flags = module['flags']
            if 'fatal' in flags or 'important' in flags:
#.........这里部分代码省略.........

            print(report)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Publish report on staging status as comment on staging project')
    parser.add_argument('-s', '--staging', type=str, default=None,
                        help='staging project')
    parser.add_argument('-f', '--force', action='store_true', default=False,
                        help='force a comment to be written')
    parser.add_argument('-p', '--project', type=str, default='openSUSE:Factory',
                        help='project to check (ex. openSUSE:Factory, openSUSE:Leap:15.1)')
    parser.add_argument('-d', '--debug', action='store_true', default=False,
                        help='enable debug information')

    args = parser.parse_args()

    osc.conf.get_config()
    osc.conf.config['debug'] = args.debug

    apiurl = osc.conf.config['apiurl']
    Config(apiurl, args.project)
    api = StagingAPI(apiurl, args.project)
    staging_report = StagingReport(api)

    if args.staging:
        staging_report.report(api.prj_from_letter(args.staging), False, args.force)
    else:
        for staging in api.get_staging_projects():
            staging_report.report(staging, True, args.force)