Python TreeherderClient.get_job_log_url方法代码示例

# 需要导入模块: from thclient import TreeherderClient [as 别名]
# 或者: from thclient.TreeherderClient import get_job_log_url [as 别名]
class Treeherder(object):
    """Wrapper class for TreeherderClient to ease the use of its API."""

    def __init__(self, application, branch, platform, server_url=TREEHERDER_URL):
        """Create a new instance of the Treeherder class.

        :param application: The name of the application to download.
        :param branch: Name of the branch.
        :param platform: Platform of the application.
        :param server_url: The URL of the Treeherder instance to access.
        """
        self.logger = logging.getLogger(__name__)

        self.client = TreeherderClient(server_url=server_url)
        self.application = application
        self.branch = branch
        self.platform = platform

    def get_treeherder_platform(self, platform):
        """Return the internal Treeherder platform identifier.

        :param platform: Platform of the application.
        """
        try:
            return PLATFORM_MAP[platform]
        except KeyError:
            raise NotSupportedError('Platform "{}" is not supported.'.format(platform))

    def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=False):
        """Retrieve build folders for a given revision with the help of Treeherder.

        :param revision: Revision of the build to download.
        :param job_type_name: Name of the job to look for. For builds it should be
            'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`.
        :param debug_build: Download a debug build.
        """
        builds = set()

        try:
            self.logger.info('Querying {url} for list of builds for revision: {revision}'.format(
                             url=self.client.server_url, revision=revision))

            # Retrieve the option hash to filter for type of build (opt, and debug for now)
            option_hash = None
            for key, values in self.client.get_option_collection_hash().iteritems():
                for value in values:
                    if value['name'] == ('debug' if debug_build else 'opt'):
                        option_hash = key
                        break
                if option_hash:
                    break

            resultsets = self.client.get_pushes(self.branch, revision=revision)

            # Set filters to speed-up querying jobs
            kwargs = {
                'option_collection_hash': option_hash,
                'job_type_name': job_type_name,
                'exclusion_profile': False,
            }
            kwargs.update(self.get_treeherder_platform(self.platform))

            for resultset in resultsets:
                kwargs.update({'result_set_id': resultset['id']})
                jobs = self.client.get_jobs(self.branch, **kwargs)
                for job in jobs:
                    log_urls = self.client.get_job_log_url(self.branch, job_id=job['id'])
                    for log_url in log_urls:
                        if self.application in log_url['url']:
                            self.logger.debug('Found build folder: {}'.format(log_url['url']))
                            builds.update([log_url['url']])

        except Exception:
            self.logger.exception('Failure occurred when querying Treeherder for builds')

        return list(builds)

# 需要导入模块: from thclient import TreeherderClient [as 别名]
# 或者: from thclient.TreeherderClient import get_job_log_url [as 别名]
def retrieve_test_logs(repo, revision, platform='linux64',
                       cache_dir=None, use_cache=True,
                       warning_re=WARNING_RE):
    """
    Retrieves and processes the test logs for the given revision.

    Returns list of processed files.
    """
    if not cache_dir:
        cache_dir = "%s-%s-%s" % (repo, revision, platform)

    cache = logspam.cache.Cache(cache_dir, warning_re)

    cache_dir_exists = os.path.isdir(cache_dir)
    if cache_dir_exists and use_cache:
        # We already have logs for this revision.
        print "Using cached data"
        try:
            return cache.read_results()
        except logspam.cache.CacheFileNotFoundException as e:
            print "Cache file for %s not found" % warning_re
            print e

    client = TreeherderClient()
    print "getting result set"
    pushes = client.get_pushes(repo, revision=revision)
    print "pushes = client.get_pushes('%s', revision='%s')" % (repo, revision)
    print "got pushes"
    if not pushes:
        print "Failed to find %s in %s" % (revision, repo)
        return None

    print "getting jobs"
    for x in range(5):
        try:
            # option_collection_hash is just the convoluted way of specifying
            # we want a debug build.
            print "jobs = client.get_jobs('%s',result_set_id=%d, count=5000, platform='%s', option_collection_hash='%s')" % (
                    repo, pushes[0]['id'], platform, DEBUG_OPTIONHASH)
            jobs = client.get_jobs(repo,
                                   result_set_id=pushes[0]['id'],
                                   count=5000, # Just make this really large to avoid pagination
                                   platform=platform,
                                   option_collection_hash=DEBUG_OPTIONHASH,
                                   state='completed')
            break
        except requests.exceptions.ConnectionError:
            pass

    if not jobs:
        print "No jobs found for %s %s" % (revision, platform)
        import traceback
        traceback.print_exc()
        return None

    print "got jobs"

    print "getting %d job log urls" % len(jobs)
    job_ids = [ job['id'] for job in jobs ]
    print job_ids
    for x in range(5):
        logs = []
        try:
            for y in range(0, len(job_ids), 100):
                logs = logs + client.get_job_log_url(repo, job_id=job_ids[y:y+100])
            job_logs = logs
            break
        except requests.exceptions.ConnectionError, e:
            pass

# 需要导入模块: from thclient import TreeherderClient [as 别名]
# 或者: from thclient.TreeherderClient import get_job_log_url [as 别名]
class Treeherder(object):
    """Wrapper class for TreeherderClient to ease the use of its API."""

    def __init__(self, application, branch, platform, host=TREEHERDER_HOST, protocol="https"):
        """Create a new instance of the Treeherder class.

        :param application: The name of the application to download.
        :param branch: Name of the branch.
        :param platform: Platform of the application.
        :param host: The Treeherder host to make use of.
        :param protocol: The protocol for the Treeherder host.
        """
        self.logger = logging.getLogger(__name__)

        self.client = TreeherderClient(host=host, protocol=protocol)
        self.application = application
        self.branch = branch
        self.platform = platform

    def get_treeherder_platform(self, platform):
        """Return the internal Treeherder platform identifier.

        :param platform: Platform of the application.
        """
        try:
            return PLATFORM_MAP[platform]
        except KeyError:
            raise NotSupportedError('Platform "{}" is not supported.'.format(platform))

    def query_builds_by_revision(self, revision, job_type_name="Build", debug_build=False):
        """Retrieve build folders for a given revision with the help of Treeherder.

        :param revision: Revision of the build to download.
        :param job_type_name: Name of the job to look for. For builds it should be
            'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`.
        :param debug_build: Download a debug build.
        """
        builds = set()

        try:
            self.logger.info(
                "Querying {host} for list of builds for revision: {revision}".format(
                    host=self.client.host, revision=revision
                )
            )

            # Retrieve the option hash to filter for type of build (opt, and debug for now)
            option_hash = None
            for key, values in self.client.get_option_collection_hash().iteritems():
                for value in values:
                    if value["name"] == ("debug" if debug_build else "opt"):
                        option_hash = key
                        break
                if option_hash:
                    break

            resultsets = self.client.get_resultsets(self.branch, revision=revision)

            # Set filters to speed-up querying jobs
            kwargs = {"option_collection_hash": option_hash, "job_type_name": job_type_name, "exclusion_profile": False}
            kwargs.update(self.get_treeherder_platform(self.platform))

            for resultset in resultsets:
                kwargs.update({"result_set_id": resultset["id"]})
                jobs = self.client.get_jobs(self.branch, **kwargs)
                for job in jobs:
                    log_urls = self.client.get_job_log_url(self.branch, job_id=job["id"])
                    for log_url in log_urls:
                        if self.application in log_url["url"]:
                            self.logger.debug("Found build folder: {}".format(log_url["url"]))
                            builds.update([log_url["url"]])

        except Exception:
            self.logger.exception("Failure occurred when querying Treeherder for builds")

        return list(builds)