Python thclient.TreeherderRequest类代码示例

def post_collection(
    project, th_collection, status=None, expect_errors=False,
    consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials( SampleData.get_credentials() )

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    if consumer_key:
        credentials['consumer_key'] = consumer_key

    if consumer_secret:
        credentials['consumer_secret'] = consumer_secret

    tr = TreeherderRequest(
        protocol='http',
        host='localhost',
        project=project,
        oauth_key=credentials['consumer_key'],
        oauth_secret=credentials['consumer_secret']
        )

    signed_uri = tr.get_signed_uri(
        th_collection.to_json(), tr.get_uri(th_collection)
        )

    response = TestApp(application).post_json(
        str(signed_uri), params=th_collection.get_collection_data(), status=status
        )

    return response

def post_job_data(
    project, uri, data, status=None, expect_errors=False):

    # Since the uri is passed in it's not generated by the
    # treeherder request or collection and is missing the protocol
    # and host. Add those missing elements here.
    uri = 'http://localhost{0}'.format(uri)

    # Set the credentials
    OAuthCredentials.set_credentials( SampleData.get_credentials() )

    credentials = OAuthCredentials.get_credentials(project)

    tr = TreeherderRequest(
        protocol='http',
        host='localhost',
        project=project,
        oauth_key=credentials['consumer_key'],
        oauth_secret=credentials['consumer_secret']
        )

    signed_uri = tr.get_signed_uri(
        json.dumps(data), uri
        )

    response = TestApp(application).post_json(
        str(signed_uri), params=data, status=status,
        expect_errors=expect_errors
        )

    return response

    def _post_json_data(url, data):

        th_collection = data[jm.project]

        OAuthCredentials.set_credentials( SampleData.get_credentials() )
        credentials = OAuthCredentials.get_credentials(jm.project)

        tr = TreeherderRequest(
            protocol='http',
            host='localhost',
            project=jm.project,
            oauth_key=credentials['consumer_key'],
            oauth_secret=credentials['consumer_secret']
            )
        signed_uri = tr.oauth_client.get_signed_uri(
            th_collection.to_json(),
            tr.get_uri(th_collection.endpoint_base),
            "POST"
            )

        response = TestApp(application).post_json(
            str(signed_uri), params=th_collection.get_collection_data()
            )

        response.getcode = lambda: response.status_int
        return response

def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    credentials = OAuthCredentials.get_credentials(project)
    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        update_parse_status(req, job_log_url, 'failed')
        if isinstance(e, urllib2.HTTPError) and e.code == 404:
            logger.debug("Log not found for %s", log_description)
            return
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact({
            "job_guid": artifact[0],
            "name": artifact[1],
            "type": artifact[2],
            "blob": artifact[3]
        })
        tac.add(ta)

    try:
        req.post(tac)
        update_parse_status(req, job_log_url, 'parsed')
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

    def post_request(self, job_collection):
        self.worker.loggerdeco.debug('AutophoneTreeherder.post_request: %s' % job_collection.__dict__)
        if not self.url or not self.worker.build.revision_hash:
            self.worker.loggerdeco.debug('AutophoneTreeherder.post_request: no url/revision hash')
            return

        req = TreeherderRequest(
            protocol=self.protocol,
            host=self.server,
            project=self.worker.build.tree,
            oauth_key=self.credentials[self.worker.build.tree]['consumer_key'],
            oauth_secret=self.credentials[self.worker.build.tree]['consumer_secret']
            )

        try:
            for attempt in range(1, self.retries+1):
                response = req.post(job_collection)
                self.worker.loggerdeco.debug('AutophoneTreeherder.post_request attempt %d: '
                                             'body: %s headers: %s msg: %s status: %s '
                                             'reason: %s' % (
                                                 attempt,
                                                 response.read(),
                                                 response.getheaders(),
                                                 response.msg,
                                                 response.status,
                                                 response.reason))
                if response.reason == 'OK':
                    break
                msg = ('Attempt %d to post result to Treeherder failed.\n\n'
                       'Response:\n'
                       'body: %s\n'
                       'headers: %s\n'
                       'msg: %s\n'
                       'status: %s\n'
                       'reason: %s\n' % (
                           attempt,
                           response.read(), response.getheaders(),
                           response.msg, response.status,
                           response.reason))
                self.worker.loggerdeco.error(msg)
                self.worker.mailer.send('Attempt %d for Phone %s failed to post to Treeherder' %
                                        (attempt, self.worker.phone.id), msg)
                time.sleep(self.retry_wait)
        except Exception, e:
            self.worker.loggerdeco.exception('Error submitting request to Treeherder')
            self.worker.mailer.send('Error submitting request to Treeherder',
                                    'Phone: %s\n'
                                    'TreeherderClientError: %s\n'
                                    'TreeherderJobCollection %s\n' % (
                                        self.worker.phone.id,
                                        e,
                                        job_collection.to_json()))

    def test_send_without_oauth(
        self, mock_HTTPConnection, mock_time, mock_generate_nonce):

        """Can send data to the server."""
        mock_time.return_value = 1342229050
        mock_generate_nonce.return_value = "46810593"

        host = 'host'

        req = TreeherderRequest(
            protocol='http',
            host=host,
            project='project',
            oauth_key=None,
            oauth_secret=None,
            )

        mock_conn = mock_HTTPConnection.return_value
        mock_request = mock_conn.request
        mock_response = mock_conn.getresponse.return_value

        tjc = TreeherderJobCollection()

        for job in self.job_data:

            tjc.add( tjc.get_job(job) )
            break

        response = req.post(tjc)

        self.assertEqual(mock_HTTPConnection.call_count, 1)
        self.assertEqual(mock_HTTPConnection.call_args[0][0], host)
        self.assertEqual(mock_response, response)
        self.assertEqual(mock_request.call_count, 1)

        uri = req.get_uri(tjc)

        method, path, data, header = mock_request.call_args[0]
        self.assertEqual(method, "POST")

        deserialized_data = json.loads(data)
        self.assertEqual(
            deserialized_data,
            tjc.get_collection_data()
            )

        self.assertEqual(
            header['Content-Type'],
            'application/json',
            )

    def load(self, th_collections):

        for project in th_collections:

            credentials = OAuthCredentials.get_credentials(project)

            th_request = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None)
            )

            response = th_request.post(th_collections[project])

            if not response or response.status != 200:
                message = response.read()
                logger.error("collection loading failed: {0}".format(message))

    def load(self, th_collections):
        for project in th_collections:

            credentials = OAuthCredentials.get_credentials(project)

            th_request = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None)
            )

            logger.info(
                "collection loading request: {0}".format(
                    th_request.get_uri(th_collections[project].endpoint_base)))
            response = th_request.post(th_collections[project])

            if not response or response.status != 200:
                message = response.read()
                logger.error('[{0}]Error posting data to {1} : {2}'.format(
                    project, th_collections[project].endpoint_base, message))

    def test_send_artifact_collection(self, mock_send):
        """Can add a artifact collections to a TreeherderRequest."""

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:

            tac.add(tac.get_artifact(artifact))

        req = TreeherderRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
        )

        req.post(tac)

        self.assertEqual(mock_send.call_count, 1)
        self.assertEqual(
            tac.to_json(),
            mock_send.call_args_list[0][1]["data"]
        )

    def test_send_result_collection(self, mock_send):
        """Can add a treeherder collections to a TreeherderRequest."""

        trc = TreeherderResultSetCollection()

        for resultset in self.resultset_data:

            trc.add( trc.get_resultset(resultset) )

        req = TreeherderRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
            )

        req.post(trc)

        self.assertEqual(mock_send.call_count, 1)
        self.assertEqual(
            trc.to_json(),
            mock_send.call_args_list[0][1]['data']
            )

    def test_send_job_collection(self, mock_send):
        """Can add a treeherder collections to a TreeherderRequest."""

        tjc = TreeherderJobCollection()

        for job in self.job_data:

            tjc.add( tjc.get_job(job) )

        req = TreeherderRequest(
            protocol='http',
            host='host',
            project='project',
            oauth_key='key',
            oauth_secret='secret',
            )

        req.post(tjc)

        self.assertEqual(mock_send.call_count, 1)
        self.assertEqual(
            tjc.to_json(),
            mock_send.call_args_list[0][1]['data']
            )

#.........这里部分代码省略.........
        lookup_url = urljoin(
            self.treeherder_url,
            'api/project/%s/revision-lookup/?revision=%s' % (
                project, revision))
        self.logger.debug('Getting revision hash from: %s' % lookup_url)
        response = requests.get(lookup_url)
        response.raise_for_status()
        assert response.json(), 'Unable to determine revision hash for %s. ' \
                                'Perhaps it has not been ingested by ' \
                                'Treeherder?' % revision
        revision_hash = response.json()[revision]['revision_hash']
        job.add_revision_hash(revision_hash)
        job.add_project(project)
        job.add_job_guid(str(uuid.uuid4()))
        job.add_product_name('b2g')
        job.add_state('completed')

        # Determine test result
        if self.failed or self.unexpected_successes:
            job.add_result('testfailed')
        else:
            job.add_result('success')

        job.add_submit_timestamp(int(self.start_time))
        job.add_start_timestamp(int(self.start_time))
        job.add_end_timestamp(int(self.end_time))

        job.add_machine(socket.gethostname())
        job.add_build_info('b2g', 'b2g-device-image', 'x86')
        job.add_machine_info('b2g', 'b2g-device-image', 'x86')

        # All B2G device builds are currently opt builds
        job.add_option_collection({'opt': True})

        # TODO: Add log reference
        # job.add_log_reference()

        date_format = '%d %b %Y %H:%M:%S'
        job_details = [{
            'content_type': 'link',
            'title': 'Gaia revision:',
            'url': 'https://github.com/mozilla-b2g/gaia/commit/%s' %
                   version.get('gaia_changeset'),
            'value': version.get('gaia_changeset'),
        }, {
            'content_type': 'text',
            'title': 'Gaia date:',
            'value': version.get('gaia_date') and time.strftime(
                date_format, time.localtime(int(version.get('gaia_date')))),
        }, {
            'content_type': 'text',
            'title': 'Device identifier:',
            'value': version.get('device_id')
        }, {
            'content_type': 'text',
            'title': 'Device firmware (date):',
            'value': version.get('device_firmware_date') and time.strftime(
                date_format, time.localtime(int(
                    version.get('device_firmware_date')))),
        }, {
            'content_type': 'text',
            'title': 'Device firmware (incremental):',
            'value': version.get('device_firmware_version_incremental')
        }, {
            'content_type': 'text',
            'title': 'Device firmware (release):',
            'value': version.get('device_firmware_version_release')
        }]

        if self.ci_url:
            job_details.append({
                'url': self.ci_url,
                'value': self.ci_url,
                'content_type': 'link',
                'title': 'CI build:'})

        if job_details:
            job.add_artifact('Job Info', 'json', {'job_details': job_details})

        # TODO: Add XML/HTML reports as artifacts
        # job.add_artifact()

        job_collection.add(job)

        # Send the collection to Treeherder
        url = urlparse(self.treeherder_url)
        request = TreeherderRequest(
            protocol=url.scheme,
            host=url.hostname,
            project=project,
            oauth_key=self.treeherder_key,
            oauth_secret=self.treeherder_secret)
        self.logger.debug('Sending results to Treeherder: %s' %
                          job_collection.to_json())
        response = request.post(job_collection)
        self.logger.debug('Response: %s' % response.read())
        assert response.status == 200, 'Failed to send results!'
        self.logger.info('Results are available to view at: %s' % (
            urljoin(self.treeherder_url, '/ui/#/jobs?repo=%s&revision=%s' % (
                project, revision))))

#.........这里部分代码省略.........
                "value": version.get("gaia_changeset"),
            },
            {
                "content_type": "text",
                "title": "Gaia date:",
                "value": version.get("gaia_date")
                and time.strftime(date_format, time.localtime(int(version.get("gaia_date")))),
            },
            {"content_type": "text", "title": "Device identifier:", "value": version.get("device_id")},
            {
                "content_type": "text",
                "title": "Device firmware (date):",
                "value": version.get("device_firmware_date")
                and time.strftime(date_format, time.localtime(int(version.get("device_firmware_date")))),
            },
            {
                "content_type": "text",
                "title": "Device firmware (incremental):",
                "value": version.get("device_firmware_version_incremental"),
            },
            {
                "content_type": "text",
                "title": "Device firmware (release):",
                "value": version.get("device_firmware_version_release"),
            },
        ]

        ci_url = os.environ.get("BUILD_URL")
        if ci_url:
            job_details.append({"url": ci_url, "value": ci_url, "content_type": "link", "title": "CI build:"})

        # Attach logcat
        adb_device = ADBDevice(self.device_serial)
        with tempfile.NamedTemporaryFile(suffix="logcat.txt") as f:
            f.writelines(adb_device.get_logcat())
            self.logger.debug("Logcat stored in: %s" % f.name)
            try:
                url = self.upload_to_s3(f.name)
                job_details.append({"url": url, "value": "logcat.txt", "content_type": "link", "title": "Log:"})
            except S3UploadError:
                job_details.append({"value": "Failed to upload logcat.txt", "content_type": "text", "title": "Error:"})

        # Attach log files
        handlers = [
            handler
            for handler in self.logger.handlers
            if isinstance(handler, StreamHandler) and os.path.exists(handler.stream.name)
        ]
        for handler in handlers:
            path = handler.stream.name
            filename = os.path.split(path)[-1]
            try:
                url = self.upload_to_s3(path)
                job_details.append({"url": url, "value": filename, "content_type": "link", "title": "Log:"})
                # Add log reference
                if (
                    type(handler.formatter) is TbplFormatter
                    or type(handler.formatter) is LogLevelFilter
                    and type(handler.formatter.inner) is TbplFormatter
                ):
                    job.add_log_reference(filename, url)
            except S3UploadError:
                job_details.append(
                    {"value": "Failed to upload %s" % filename, "content_type": "text", "title": "Error:"}
                )

        # Attach reports
        for report in [self.html_output]:
            if report is not None:
                filename = os.path.split(report)[-1]
                try:
                    url = self.upload_to_s3(report)
                    job_details.append({"url": url, "value": filename, "content_type": "link", "title": "Report:"})
                except S3UploadError:
                    job_details.append(
                        {"value": "Failed to upload %s" % filename, "content_type": "text", "title": "Error:"}
                    )

        if job_details:
            job.add_artifact("Job Info", "json", {"job_details": job_details})

        job_collection.add(job)

        # Send the collection to Treeherder
        url = urlparse(self.treeherder_url)
        request = TreeherderRequest(
            protocol=url.scheme,
            host=url.hostname,
            project=project,
            oauth_key=os.environ.get("TREEHERDER_KEY"),
            oauth_secret=os.environ.get("TREEHERDER_SECRET"),
        )
        self.logger.debug("Sending results to Treeherder: %s" % job_collection.to_json())
        response = request.post(job_collection)
        self.logger.debug("Response: %s" % response.read())
        assert response.status == 200, "Failed to send results!"
        self.logger.info(
            "Results are available to view at: %s"
            % (urljoin(self.treeherder_url, "/ui/#/jobs?repo=%s&revision=%s" % (project, revision)))
        )

#.........这里部分代码省略.........

        # Attach log files
        handlers = [handler for handler in self._logger.handlers
                    if isinstance(handler, StreamHandler) and
                    os.path.exists(handler.stream.name)]
        for handler in handlers:
            path = handler.stream.name
            filename = os.path.split(path)[-1]
            try:
                url = self.upload_to_s3(path)
                job_details.append({
                    'url': url,
                    'value': filename,
                    'content_type': 'link',
                    'title': 'Log:'})
                # Add log reference
                if type(handler.formatter) is TbplFormatter or \
                        type(handler.formatter) is LogLevelFilter and \
                        type(handler.formatter.inner) is TbplFormatter:
                    job.add_log_reference(filename, url)
            except S3UploadError:
                job_details.append({
                    'value': 'Failed to upload %s' % filename,
                    'content_type': 'text',
                    'title': 'Error:'})

        # Attach script
        filename = os.path.split(script)[-1]
        try:
            url = self.upload_to_s3(script)
            job_details.append({
                'url': url,
                'value': filename,
                'content_type': 'link',
                'title': 'Script:'})
        except S3UploadError:
            job_details.append({
                'value': 'Failed to upload %s' % filename,
                'content_type': 'text',
                'title': 'Error:'})

        # Attach logcat
        filename = '%s.log' % self.runner.device.dm._deviceSerial
        path = os.path.join(self.temp_dir, filename)
        try:
            url = self.upload_to_s3(path)
            job_details.append({
                'url': url,
                'value': filename,
                'content_type': 'link',
                'title': 'Logcat:'})
        except S3UploadError:
            job_details.append({
                'value': 'Failed to upload %s' % filename,
                'content_type': 'text',
                'title': 'Error:'})

        if job_details:
            job.add_artifact('Job Info', 'json', {'job_details': job_details})

        # Attach crash dumps
        if self.runner.crashed:
            crash_dumps = os.listdir(self.crash_dumps_path)
            for filename in crash_dumps:
                path = os.path.join(self.crash_dumps_path, filename)
                try:
                    url = self.upload_to_s3(path)
                    job_details.append({
                        'url': url,
                        'value': filename,
                        'content_type': 'link',
                        'title': 'Crash:'})
                except S3UploadError:
                    job_details.append({
                        'value': 'Failed to upload %s' % filename,
                        'content_type': 'text',
                        'title': 'Error:'})

        job_collection.add(job)

        # Send the collection to Treeherder
        url = urlparse(treeherder_url)
        request = TreeherderRequest(
            protocol=url.scheme,
            host=url.hostname,
            project=project,
            oauth_key=os.environ.get('TREEHERDER_KEY'),
            oauth_secret=os.environ.get('TREEHERDER_SECRET'))
        self._logger.info('Sending results to Treeherder: %s' % treeherder_url)
        self._logger.debug('Job collection: %s' %
                           job_collection.to_json())
        response = request.post(job_collection)
        if response.status == 200:
            self._logger.debug('Response: %s' % response.read())
            self._logger.info('Results are available to view at: %s' % (
                urljoin(treeherder_url, '/ui/#/jobs?repo=%s&revision=%s' % (
                    project, revision))))
        else:
            self._logger.error('Failed to send results to Treeherder! '
                               'Response: %s' % response.read())

def parse_log(project, job_log_url, job_guid, check_errors=False):
    """
    Call ArtifactBuilderCollection on the given job.
    """
    credentials = OAuthCredentials.get_credentials(project)
    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )
    update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])

    try:
        log_url = job_log_url['url']
        bug_suggestions = []
        bugscache_uri = '{0}{1}'.format(
            settings.API_HOSTNAME,
            reverse("bugscache-list")
        )
        terms_requested = {}

        if log_url:
            # parse a log given its url
            artifact_bc = ArtifactBuilderCollection(log_url,
                                                    check_errors=check_errors)
            artifact_bc.parse()

            artifact_list = []
            for name, artifact in artifact_bc.artifacts.items():
                artifact_list.append((job_guid, name, 'json',
                                      json.dumps(artifact)))
            if check_errors:
                all_errors = artifact_bc.artifacts.get(
                    'Structured Log', {}
                    ).get(
                        'step_data', {}
                        ).get(
                            'all_errors', [] )

                for err in all_errors:
                    # remove the mozharness prefix
                    clean_line = get_mozharness_substring(err['line'])
                    # get a meaningful search term out of the error line
                    search_term = get_error_search_term(clean_line)
                    bugs = dict(open_recent=[], all_others=[])

                    # collect open recent and all other bugs suggestions
                    if search_term:
                        if not search_term in terms_requested:
                            # retrieve the list of suggestions from the api
                            bugs = get_bugs_for_search_term(
                                search_term,
                                bugscache_uri
                            )
                            terms_requested[search_term] = bugs
                        else:
                            bugs = terms_requested[search_term]

                    if not bugs or not (bugs['open_recent']
                                        or bugs['all_others']):
                        # no suggestions, try to use
                        # the crash signature as search term
                        crash_signature = get_crash_signature(clean_line)
                        if crash_signature:
                            if not crash_signature in terms_requested:
                                bugs = get_bugs_for_search_term(
                                    crash_signature,
                                    bugscache_uri
                                )
                                terms_requested[crash_signature] = bugs
                            else:
                                bugs = terms_requested[crash_signature]

                    bug_suggestions.append({
                        "search": clean_line,
                        "bugs": bugs
                    })

            artifact_list.append((job_guid, 'Bug suggestions', 'json', json.dumps(bug_suggestions)))

            # store the artifacts generated
            tac = TreeherderArtifactCollection()
            for artifact in artifact_list:
                ta = tac.get_artifact({
                    "job_guid": artifact[0],
                    "name": artifact[1],
                    "type": artifact[2],
                    "blob": artifact[3]
                })
                tac.add(ta)

            req.post(tac)

            # send an update to job_log_url
            # the job_log_url status changes
            # from pending to running
            current_timestamp = time.time()
            status = 'parsed'
#.........这里部分代码省略.........