Python TreeherderRequest.send方法代码示例

# 需要导入模块: from thclient import TreeherderRequest [as 别名]
# 或者: from thclient.TreeherderRequest import send [as 别名]
    def load(self, th_collections):

        for project in th_collections:

            credentials = OAuthCredentials.get_credentials(project)

            th_request = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None)
            )

            response = th_request.send(th_collections[project])

            if not response or response.status != 200:
                message = response.read()
                logger.error("collection loading failed: {0}".format(message))

# 需要导入模块: from thclient import TreeherderRequest [as 别名]
# 或者: from thclient.TreeherderRequest import send [as 别名]
def parse_log(project, job_log_url, job_guid, check_errors=False):
    """
    Call ArtifactBuilderCollection on the given job.
    """

    # if parse_status is not available, consider it pending
    parse_status = job_log_url.get("parse_status", "pending")
    # don't parse a log if it's already been parsed
    if parse_status == "parsed":
        return

    try:
        credentials = OAuthCredentials.get_credentials(project)
        req = TreeherderRequest(
            protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
            host=settings.TREEHERDER_REQUEST_HOST,
            project=project,
            oauth_key=credentials.get('consumer_key', None),
            oauth_secret=credentials.get('consumer_secret', None),
        )
        update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
            job_log_url['id']
        )

        artifact_list = extract_log_artifacts(job_log_url['url'],
                                              job_guid, check_errors)
        # store the artifacts generated
        tac = TreeherderArtifactCollection()
        for artifact in artifact_list:
            ta = tac.get_artifact({
                "job_guid": artifact[0],
                "name": artifact[1],
                "type": artifact[2],
                "blob": artifact[3]
            })
            tac.add(ta)

        req.post(tac)

        # send an update to job_log_url
        # the job_log_url status changes from pending to parsed
        current_timestamp = time.time()
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': 'parsed',
                'parse_timestamp': current_timestamp
            }
        )
    except Exception, e:
        # send an update to job_log_url
        #the job_log_url status changes from pending/running to failed
        current_timestamp = time.time()
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': 'failed',
                'parse_timestamp': current_timestamp
            }
        )
        # for every retry, set the countdown to 10 minutes
        # .retry() raises a RetryTaskError exception,
        # so nothing below this line will be executed.
        parse_log.retry(exc=e, countdown=10*60)

# 需要导入模块: from thclient import TreeherderRequest [as 别名]
# 或者: from thclient.TreeherderRequest import send [as 别名]

#.........这里部分代码省略.........
                                                    check_errors=check_errors)
            artifact_bc.parse()

            artifact_list = []
            for name, artifact in artifact_bc.artifacts.items():
                artifact_list.append((job_guid, name, 'json',
                                      json.dumps(artifact)))
            if check_errors:
                all_errors = artifact_bc.artifacts.get(
                    'Structured Log', {}
                    ).get(
                        'step_data', {}
                        ).get(
                            'all_errors', [] )

                for err in all_errors:
                    # remove the mozharness prefix
                    clean_line = get_mozharness_substring(err['line'])
                    # get a meaningful search term out of the error line
                    search_term = get_error_search_term(clean_line)
                    bugs = dict(open_recent=[], all_others=[])

                    # collect open recent and all other bugs suggestions
                    if search_term:
                        if not search_term in terms_requested:
                            # retrieve the list of suggestions from the api
                            bugs = get_bugs_for_search_term(
                                search_term,
                                bugscache_uri
                            )
                            terms_requested[search_term] = bugs
                        else:
                            bugs = terms_requested[search_term]

                    if not bugs or not (bugs['open_recent']
                                        or bugs['all_others']):
                        # no suggestions, try to use
                        # the crash signature as search term
                        crash_signature = get_crash_signature(clean_line)
                        if crash_signature:
                            if not crash_signature in terms_requested:
                                bugs = get_bugs_for_search_term(
                                    crash_signature,
                                    bugscache_uri
                                )
                                terms_requested[crash_signature] = bugs
                            else:
                                bugs = terms_requested[crash_signature]

                    bug_suggestions.append({
                        "search": clean_line,
                        "bugs": bugs
                    })

            artifact_list.append((job_guid, 'Bug suggestions', 'json', json.dumps(bug_suggestions)))

            # store the artifacts generated
            tac = TreeherderArtifactCollection()
            for artifact in artifact_list:
                ta = tac.get_artifact({
                    "job_guid": artifact[0],
                    "name": artifact[1],
                    "type": artifact[2],
                    "blob": artifact[3]
                })
                tac.add(ta)

            req.post(tac)

            # send an update to job_log_url
            # the job_log_url status changes
            # from pending to running
            current_timestamp = time.time()
            status = 'parsed'
            req.send(
                update_endpoint,
                method='POST',
                data={
                    'parse_status': status,
                    'parse_timestamp': current_timestamp
                }
            )

    except Exception, e:
        parse_log.retry(exc=e)
        # send an update to job_log_url
        # the job_log_url status changes
        # from pending to running
        current_timestamp = time.time()
        status = 'failed'
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': status,
                'parse_timestamp': current_timestamp
            }
        )
        # re raise the exception to leave a trace in the log
        raise

# 需要导入模块: from thclient import TreeherderRequest [as 别名]
# 或者: from thclient.TreeherderRequest import send [as 别名]
def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    credentials = OAuthCredentials.get_credentials(project)
    update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
        job_log_url['id']
    )

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        current_timestamp = time.time()
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': 'failed',
                'parse_timestamp': current_timestamp
            }
        )
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact({
            "job_guid": artifact[0],
            "name": artifact[1],
            "type": artifact[2],
            "blob": artifact[3]
        })
        tac.add(ta)

    try:
        req.post(tac)

        # send an update to job_log_url
        # the job_log_url status changes from pending to parsed
        current_timestamp = time.time()
        req.send(
            update_endpoint,
            method='POST',
            data={
                'parse_status': 'parsed',
                'parse_timestamp': current_timestamp
            }
        )
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

# 需要导入模块: from thclient import TreeherderRequest [as 别名]
# 或者: from thclient.TreeherderRequest import send [as 别名]
def parse_log(project, log_url, job_guid, resultset, check_errors=False):
    """
    Call ArtifactBuilderCollection on the given job.
    """
    mozharness_pattern = re.compile(
        '^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
    )

    bugs_cache = {'open': {}, 'closed': {}}
    bug_suggestions = {'open': {}, 'closed': {}}

    status_publisher = JobStatusPublisher(settings.BROKER_URL)
    failure_publisher = JobFailurePublisher(settings.BROKER_URL)

    try:
        # return the resultset with the job id to identify if the UI wants
        # to fetch the whole thing.

        bugscache_uri = '{0}{1}'.format(
            settings.API_HOSTNAME,
            reverse("bugscache-list")
        )

        credentials = OAuthCredentials.get_credentials(project)

        if log_url:
            # parse a log given its url
            artifact_bc = ArtifactBuilderCollection(
                log_url,
                check_errors=check_errors,
            )
            artifact_bc.parse()

            artifact_list = []
            for name, artifact in artifact_bc.artifacts.items():
                artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))

            if check_errors:
                all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
                for err in all_errors:
                    # remove the mozharness prefix
                    clean_line = mozharness_pattern.sub('', err['line']).strip()
                    # get a meaningful search term out of the error line
                    search_term = get_error_search_term(clean_line)
                    # collect open and closed bugs suggestions
                    for status in ('open', 'closed'):
                        if not search_term:
                            bug_suggestions[status][clean_line] = []
                            continue
                        if search_term not in bugs_cache[status]:
                            # retrieve the list of suggestions from the api
                            bugs_cache[status][search_term] = get_bugs_for_search_term(
                                search_term,
                                status,
                                bugscache_uri
                            )
                            # no suggestions, try to use the crash signature as search term
                            if not bugs_cache[status][search_term]:
                                crash_signature = get_crash_signature(search_term)
                                if crash_signature:
                                    bugs_cache[status][search_term] = get_bugs_for_search_term(
                                        search_term,
                                        status,
                                        bugscache_uri
                                    )
                        bug_suggestions[status][clean_line] = bugs_cache[status][search_term]

                artifact_list.append((job_guid, 'Open bugs', 'json', json.dumps(bug_suggestions['open'])))
                artifact_list.append((job_guid, 'Closed bugs', 'json', json.dumps(bug_suggestions['closed'])))

            # store the artifacts generated
            tac = TreeherderArtifactCollection()
            for artifact in artifact_list:
                ta = tac.get_artifact({
                    "job_guid": artifact[0],
                    "name": artifact[1],
                    "type": artifact[2],
                    "blob": artifact[3]
                })
                tac.add(ta)
            req = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None),
            )
            req.send(tac)

        status_publisher.publish(job_guid, resultset, project, 'processed')
        if check_errors:
            failure_publisher.publish(job_guid, project)

    finally:
        status_publisher.disconnect()
        failure_publisher.disconnect()