Python thclient.TreeherderJobCollection类代码示例

    def submit(self, job):
        """Submit the job to treeherder.

        :param job: Treeherder job instance to use for submission.

        """
        job.add_submit_timestamp(int(time.time()))

        # We can only submit job info once, so it has to be done in completed
        if self._job_details:
            job.add_artifact('Job Info', 'json', {'job_details': self._job_details})

        job_collection = TreeherderJobCollection()
        job_collection.add(job)

        logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
        url = urlparse(self.url)
        client = TreeherderClient(protocol=url.scheme, host=url.hostname,
                                  client_id=self.client_id, secret=self.secret)
        client.post_collection(self.repository, job_collection)

        logger.info('Results are available to view at: {}'.format(
                    urljoin(self.url,
                            JOB_FRAGMENT.format(repository=self.repository,
                                                revision=self.revision))))

def post_treeherder_jobs(client, fileNames):
    """
    Processes each file and submits a treeherder job with the data from each file.

    :param client: The TreeherderClient to use.
    :param fileNames: The files to process.
    """
    for name in fileNames:
        with gzip.open(name) as f:
            data = json.load(f)

        test_set = data['Slimtest-TalosTP5-Slow']

        nodes = test_set['nodes']
        repo = test_set.get('repo', 'mozilla-inbound')
        # Attempt to retrieve the revision from the metadata, otherwise parse
        # it from the file name which has the form <revision>.json.gz
        if 'revision' in test_set:
            revsion = test_set['revision']
        else:
            revision = os.path.basename(name).split('.')[0]

        tjc = TreeherderJobCollection()
        try:
            tjc.add(create_treeherder_job(repo, revision, client, nodes))
        except KeyError as e:
            print "Failed to generate data for %s: %s" % (revision, e)
            continue

        # NB: In theory we could batch these, but each collection has to be from
        #     the same repo and it's possible we have different repos in our
        #     dataset.
        client.post_collection(repo, tjc)

def post_treeherder_jobs(client, revisions, s3=None):
    """
    Processes each file and submits a treeherder job with the data from each file.

    :param client: The TreeherderClient to use.
    :param revisions: A dictionary of revisions and their associated data.
    :param s3: Optional Amazon S3 bucket to upload logs to.
    """
    successful = []
    for (revision, test_set) in revisions.iteritems():
        nodes = test_set['nodes']
        repo = test_set.get('repo', 'mozilla-inbound')

        tjc = TreeherderJobCollection()
        try:
            tjc.add(create_treeherder_job(repo, revision, client, nodes, s3))
        except KeyError as e:
            print "Failed to generate data for %s: %s, probably still running" % (revision, e)
            continue

        try:
            # NB: In theory we could batch these, but each collection has to be from
            #     the same repo and it's possible we have different repos in our
            #     dataset.
            client.post_collection(repo, tjc)
            #print tjc.to_json()

            successful.append(revision)
            print "Submitted perf data for %s to %s" % (revision, client.server_url)
        except Exception as e:
            print "Failed to submit data for %s: %s" % (revision, e)

    return successful

def test_objectstore_create(job_sample, jm):
    """
    test posting data to the objectstore via webtest.
    extected result are:
    - return code 200
    - return message successful
    - 1 job stored in the objectstore
    """

    tjc = TreeherderJobCollection()
    tj = tjc.get_job(job_sample)
    tjc.add(tj)

    resp = test_utils.post_collection(jm.project, tjc)

    assert resp.status_int == 200
    assert resp.json['message'] == 'well-formed JSON stored'

    stored_objs = jm.get_os_dhub().execute(
        proc="objectstore_test.selects.row_by_guid",
        placeholders=[job_sample["job"]["job_guid"]]
    )

    assert len(stored_objs) == 1

    assert stored_objs[0]['job_guid'] == job_sample["job"]["job_guid"]

    def submit_pending(self, jobs):
        """Submit jobs pending notifications to Treeherder
        :param jobs: Lists of jobs to be reported. (TestJob)
        """
        self.logger.debug(type(self).__name__ +
                          '.submit_pending: jobs =\n%s' % jobs)
        if not self.url or not jobs:
            self.logger.debug(type(self).__name__ +
                              '.submit_pending: no url/job')
            return

        tjc = TreeherderJobCollection()

        for j in jobs:
            project = j.build['repo']
            revision = j.build['revision']
            revision_hash = self.request_revision_hash(project, revision)
            if not revision_hash:
                self.logger.debug(type(self).__name__ +
                                  '.submit_pending: no revision hash')
                return
            j.submit_timestamp = timestamp_now()

            self.logger.info('creating Treeherder job %s for %s %s, '
                             'revision_hash: %s' % (j.job_guid,
                                                    j.name, project,
                                                    revision_hash))

            tj = tjc.get_job()
            tj.add_description(j.description)
            tj.add_reason(j.reason)
            tj.add_tier(self.tier)
            tj.add_revision_hash(revision_hash)
            tj.add_project(project)
            tj.add_who(j.who)
            tj.add_job_guid(j.job_guid)
            tj.add_job_name(j.job_name)
            tj.add_job_symbol(j.job_symbol)
            tj.add_group_name(j.group_name)
            tj.add_group_symbol(j.group_symbol)
            tj.add_product_name(j.build['product'])
            tj.add_state(JobState.PENDING)
            tj.add_submit_timestamp(j.submit_timestamp)
            # XXX need to send these until Bug 1066346 fixed.
            tj.add_start_timestamp(j.submit_timestamp)
            tj.add_end_timestamp(j.submit_timestamp)
            tj.add_build_url(j.build_url)
            tj.add_build_info(j.build['os_name'],
                              j.build['platform'],
                              j.build['architecture'])
            tj.add_machine(j.machine['host'])
            tj.add_machine_info(j.machine['os_name'],
                                j.machine['platform'],
                                j.machine['architecture'])
            # TODO determine type of build
            tj.add_option_collection({'opt': True})

            tjc.add(tj)

        self.post_request(project, tjc, j.job_guid)

 def serve_forever(self):
     logger = utils.getLogger()
     while not self.shutdown_requested:
         wait_seconds = 1    # avoid busy loop
         job = self.jobs.get_next_treeherder_job()
         if job:
             tjc = TreeherderJobCollection()
             for data in job['job_collection']:
                 tj = TreeherderJob(data)
                 tjc.add(tj)
             if self.post_request(job['machine'], job['project'], tjc,
                                  job['attempts'], job['last_attempt']):
                 self.jobs.treeherder_job_completed(job['id'])
                 wait_seconds = 0
             else:
                 attempts = int(job['attempts'])
                 wait_seconds = min(self.retry_wait * attempts, 3600)
                 logger.debug('AutophoneTreeherder waiting for %d seconds after '
                              'failed attempt %d',
                              wait_seconds, attempts)
         if wait_seconds > 0:
             for i in range(wait_seconds):
                 if self.shutdown_requested:
                     break
                 time.sleep(1)

    def submit_running(self, machine, build_url, project, revision_hash, tests=None):
        """Submit tests running notifications to Treeherder

        :param machine: machine id
        :param build_url: url to build being tested.
        :param project: repository of build.
        :param revision_hash: Treeherder revision hash of build.
        :param tests: Lists of tests to be reported.
        """
        if tests is None:
            tests = []
        logger.debug('AutophoneTreeherder.submit_running: %s' % tests)
        if not self.url or not revision_hash:
            logger.debug('AutophoneTreeherder.submit_running: no url/revision hash')
            return

        tjc = TreeherderJobCollection()

        for t in tests:
            logger.debug('AutophoneTreeherder.submit_running: '
                         'for %s %s' % (t.name, project))

            t.submit_timestamp = timestamp_now()
            t.start_timestamp = timestamp_now()

            tj = tjc.get_job()
            tj.add_tier(self.options.treeherder_tier)
            tj.add_revision_hash(revision_hash)
            tj.add_project(project)
            tj.add_job_guid(t.job_guid)
            tj.add_job_name(t.job_name)
            tj.add_job_symbol(t.job_symbol)
            tj.add_group_name(t.group_name)
            tj.add_group_symbol(t.group_symbol)
            tj.add_product_name('fennec')
            tj.add_state(TestState.RUNNING)
            tj.add_submit_timestamp(t.submit_timestamp)
            tj.add_start_timestamp(t.start_timestamp)
            # XXX need to send these until Bug 1066346 fixed.
            tj.add_end_timestamp(0)
            #
            tj.add_machine(machine)
            tj.add_build_info('android', t.phone.platform, t.phone.architecture)
            tj.add_machine_info('android',t.phone.platform, t.phone.architecture)
            tj.add_option_collection({'opt': True})

            tj.add_artifact('buildapi', 'json', {
                'buildername': t.get_buildername(project)})
            tj.add_artifact('privatebuild', 'json', {
                'build_url': build_url,
                'config_file': t.config_file,
                'chunk': t.chunk})
            tjc.add(tj)

        logger.debug('AutophoneTreeherder.submit_running: tjc: %s' %
                     tjc.to_json())

        self.queue_request(machine, project, tjc)

def create_job_collection(dataset):
    print("[DEBUG] Job Collection:")
    print(dataset)

    tjc = TreeherderJobCollection()

    for data in dataset:

        tj = tjc.get_job()

        tj.add_revision(data['revision'])
        tj.add_project(data['project'])
        tj.add_coalesced_guid(data['job']['coalesced'])
        tj.add_job_guid(data['job']['job_guid'])
        tj.add_job_name(data['job']['name'])
        tj.add_job_symbol(data['job']['job_symbol'])
        tj.add_group_name(data['job']['group_name'])
        tj.add_group_symbol(data['job']['group_symbol'])
        tj.add_description(data['job']['desc'])
        tj.add_product_name(data['job']['product_name'])
        tj.add_state(data['job']['state'])
        tj.add_result(data['job']['result'])
        tj.add_reason(data['job']['reason'])
        tj.add_who(data['job']['who'])
        tj.add_tier(data['job']['tier'])
        tj.add_submit_timestamp(data['job']['submit_timestamp'])
        tj.add_start_timestamp(data['job']['start_timestamp'])
        tj.add_end_timestamp(data['job']['end_timestamp'])
        tj.add_machine(data['job']['machine'])

        tj.add_build_info(
            data['job']['build_platform']['os_name'],
            data['job']['build_platform']['platform'],
            data['job']['build_platform']['architecture']
        )

        tj.add_machine_info(
            data['job']['machine_platform']['os_name'],
            data['job']['machine_platform']['platform'],
            data['job']['machine_platform']['architecture']
        )

        tj.add_option_collection(data['job']['option_collection'])

        # for log_reference in data['job']['log_references']:
        #    tj.add_log_reference( 'buildbot_text', log_reference['url'])

        # data['artifact'] is a list of artifacts
        for artifact_data in data['job']['artifacts']:
            tj.add_artifact(
                artifact_data['name'],
                artifact_data['type'],
                artifact_data['blob']
            )
        tjc.add(tj)

        return tjc

    def test_job_collection(self):
        """Confirm the collection matches the sample data"""
        tjc = TreeherderJobCollection()

        for job in self.job_data:
            tj = TreeherderJob(job)
            tjc.add(tj)

        self.assertTrue( len(self.job_data) == len(tjc.data) )

    def submit_pending(self, tests=[]):
        self.worker.loggerdeco.debug('AutophoneTreeherder.submit_pending: %s' % tests)
        if not self.url or not self.worker.build.revision_hash:
            self.worker.loggerdeco.debug('AutophoneTreeherder.submit_pending: no url/revision hash')
            return

        tjc = TreeherderJobCollection(job_type='update')

        if not tests:
            tests = self.worker.runnable_tests

        for t in tests:
            t.message = None
            t.submit_timestamp = timestamp_now()
            t.job_guid = generate_guid()
            t.job_details = []

            self.worker.loggerdeco.info('creating Treeherder job %s for %s %s, '
                                        'revision: %s, revision_hash: %s' % (
                                            t.job_guid, t.name, t.build.tree,
                                            t.build.revision, t.build.revision_hash))

            self.worker.loggerdeco.debug('AutophoneTreeherder.submit_pending: '
                                         'test config_file=%s, config sections=%s' % (
                                             t.config_file, t.cfg.sections()))

            tj = tjc.get_job()
            tj.add_revision_hash(self.worker.build.revision_hash)
            tj.add_project(self.worker.build.tree)
            tj.add_job_guid(t.job_guid)
            tj.add_job_name(t.job_name)
            tj.add_job_symbol(t.job_symbol)
            tj.add_group_name(t.group_name)
            tj.add_group_symbol(t.group_symbol)
            tj.add_product_name('fennec')
            tj.add_state(TestState.PENDING)
            tj.add_submit_timestamp(t.submit_timestamp)
            # XXX need to send these until Bug 1066346 fixed.
            tj.add_start_timestamp(t.submit_timestamp)
            tj.add_end_timestamp(t.submit_timestamp)
            #
            tj.add_machine(t.phone.id)
            tj.add_build_url(self.worker.build.url)
            tj.add_build_info('android', t.phone.platform, t.phone.architecture)
            tj.add_machine_info('android',t.phone.platform, t.phone.architecture)
            tj.add_option_collection({'opt': True})

            # Fake the buildername from buildbot...
            tj.add_artifact('buildapi', 'json', {'buildername': t.buildername})

            tjc.add(tj)

        self.worker.loggerdeco.debug('AutophoneTreeherder.submit_pending: tjc: %s' % (
            tjc.to_json()))

        self.post_request(tjc)

def running_jobs_stored(
        jm, running_jobs, result_set_stored):
    """
    stores a list of buildapi running jobs into the objectstore
    """
    running_jobs.update(result_set_stored[0])

    tjc = TreeherderJobCollection(job_type='update')
    tj = tjc.get_job(running_jobs)
    tjc.add(tj)

    test_utils.post_collection(jm.project, tjc)

def completed_jobs_stored(
        jm, completed_jobs, result_set_stored, mock_send_request):
    """
    stores a list of buildapi completed jobs into the objectstore
    """
    completed_jobs['revision_hash'] = result_set_stored[0]['revision_hash']

    tjc = TreeherderJobCollection()
    tj = tjc.get_job(completed_jobs)
    tjc.add(tj)

    test_utils.post_collection(jm.project, tjc)

def pending_jobs_stored(
        jm, pending_jobs, result_set_stored):
    """
    stores a list of buildapi pending jobs into the jobs store
    using BuildApiTreeHerderAdapter
    """

    pending_jobs.update(result_set_stored[0])

    tjc = TreeherderJobCollection(job_type='update')
    tj = tjc.get_job(pending_jobs)
    tjc.add(tj)

    test_utils.post_collection(jm.project, tjc)

    def create_job_collection(self, dataset):
        # reference the page about tjc ttps://github.com/mozilla/treeherder/blob/master/docs/submitting_data.rst
        tjc = TreeherderJobCollection()

        for data in dataset:
            tj = tjc.get_job()

            tj.add_revision(data['revision'])
            tj.add_project(data['project'])
            tj.add_coalesced_guid(data['job']['coalesced'])
            tj.add_job_guid(data['job']['job_guid'])
            tj.add_job_name(data['job']['name'])
            tj.add_job_symbol(data['job']['job_symbol'])
            tj.add_group_name(data['job']['group_name'])
            tj.add_group_symbol(data['job']['group_symbol'])
            tj.add_description(data['job']['desc'])
            tj.add_product_name(data['job']['product_name'])
            tj.add_state(data['job']['state'])
            tj.add_result(data['job']['result'])
            tj.add_reason(data['job']['reason'])
            tj.add_who(data['job']['who'])
            tj.add_tier(data['job']['tier'])
            tj.add_submit_timestamp(data['job']['submit_timestamp'])
            tj.add_start_timestamp(data['job']['start_timestamp'])
            tj.add_end_timestamp(data['job']['end_timestamp'])
            tj.add_machine(data['job']['machine'])

            tj.add_build_info(
                data['job']['build_platform']['os_name'],
                data['job']['build_platform']['platform'],
                data['job']['build_platform']['architecture']
            )

            tj.add_machine_info(
                data['job']['machine_platform']['os_name'],
                data['job']['machine_platform']['platform'],
                data['job']['machine_platform']['architecture']
            )

            tj.add_option_collection(data['job']['option_collection'])

            # data['artifact'] is a list of artifacts
            for artifact_data in data['job']['artifacts']:
                tj.add_artifact(
                    artifact_data['name'],
                    artifact_data['type'],
                    artifact_data['blob']
                )
            tjc.add(tj)
        return tjc

    def test_send_without_oauth(
        self, mock_HTTPConnection, mock_time, mock_generate_nonce):

        """Can send data to the server."""
        mock_time.return_value = 1342229050
        mock_generate_nonce.return_value = "46810593"

        host = 'host'

        req = TreeherderRequest(
            protocol='http',
            host=host,
            project='project',
            oauth_key=None,
            oauth_secret=None,
            )

        mock_conn = mock_HTTPConnection.return_value
        mock_request = mock_conn.request
        mock_response = mock_conn.getresponse.return_value

        tjc = TreeherderJobCollection()

        for job in self.job_data:

            tjc.add( tjc.get_job(job) )
            break

        response = req.post(tjc)

        self.assertEqual(mock_HTTPConnection.call_count, 1)
        self.assertEqual(mock_HTTPConnection.call_args[0][0], host)
        self.assertEqual(mock_response, response)
        self.assertEqual(mock_request.call_count, 1)

        uri = req.get_uri(tjc)

        method, path, data, header = mock_request.call_args[0]
        self.assertEqual(method, "POST")

        deserialized_data = json.loads(data)
        self.assertEqual(
            deserialized_data,
            tjc.get_collection_data()
            )

        self.assertEqual(
            header['Content-Type'],
            'application/json',
            )