Python BBScorevars类代码示例

def get_node_info():
    r_version = get_r_version()
    osys = BBScorevars.getNodeSpec(builder_id, "OS")
    arch = BBScorevars.getNodeSpec(builder_id, "Arch")
    plat = BBScorevars.getNodeSpec(builder_id, "Platform")
    send_message({"status": "node_info", "r_version": r_version,
        "os": osys, "arch": arch, "platform": plat, "body": "node_info",
        "bioc_version": os.getenv("BBS_BIOC_VERSION")})

def update_packages_file():
    global repos
    
    
    r_version = bioc_r_map[os.getenv("BBS_BIOC_VERSION")]
    if (platform.system() == "Darwin"):
        pkg_type = BBScorevars.getNodeSpec(builder_id, "pkgType")
        if pkg_type == "mac.binary.leopard":
            os_seg = "bin/macosx/contrib/%s" % r_version
        else:
            os_seg = "bin/macosx/mavericks/contrib/%s" % r_version
    elif (platform.system() == "Linux"):
        os_seg = "src/contrib"
    else:
        os_seg = "bin/windows/contrib/%s" % r_version
    
    
    if (manifest['repository'] == 'course'):
        repos = "/loc/www/bioconductor-test.fhcrc.org/course-packages/%s" % os_seg
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/","http://bioconductor.org/")
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/course-packages"
    elif (manifest['repository'] == 'scratch'):
        repos = '/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s/%s' % (manifest['bioc_version'], os_seg)
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/scratch-repos/","http://bioconductor.org/scratch-repos/")
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s" % manifest['bioc_version']
    
    pkg_type = BBScorevars.getNodeSpec(builder_id, "pkgType")
    if pkg_type == "mac.binary.leopard":
        pkg_type = "mac.binary"
    command = \
        "%s [email protected] 'R -f %s/update-repo.R --args %s %s'" \
        % (packagebuilder_ssh_cmd, script_loc, repos, pkg_type)
    print("update packages command: ")
    print(command)
    retcode = subprocess.call(command, shell=True)
    print "retcode for update packages: %d" % retcode
    send_message({"status": "post_processing", "retcode": retcode, "body": "Updated packages list"})
    if retcode != 0:
        send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Updating packages failed.",
            "build_product": build_product, "url": url})
        sys.exit("Updating packages failed")
    if (manifest['repository'] == 'course' or manifest['repository'] == 'scratch'):
        command = "%s [email protected] \"source ~/.bash_profile && cd /home/biocadmin/bioc-test-web/bioconductor.org && rake deploy_production\"" % \
            packagebuilder_ssh_cmd
        print("sync command = ")
        print(command)
        retcode = subprocess.call(command, shell=True)
        send_message({"status": "post_processing", "retcode": retcode, "body": "Synced repository to website",
            "build_product": build_product, "url": url})
        if retcode != 0:
            send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Syncing repository failed",
                "build_product": build_product, "url": url})
            sys.exit("Sync to website failed")
        send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Post-processing complete.",
            "build_product": build_product, "url": url})

def copy_outgoing_pkgs(fresh_pkgs_subdir, source_node):
    tmp = fresh_pkgs_subdir.split("/")
    if len(tmp) != 2:
        sys.exit("ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)" % fresh_pkgs_subdir)
    node_id = tmp[0]
    node_hostname = node_id.split("-")[0]
    fileext = BBScorevars.getNodeSpec(node_hostname, 'pkgFileExt')
    fresh_pkgs_subdir = os.path.join(BBScorevars.nodes_rdir.path, fresh_pkgs_subdir)

    ## Workflow packages do not have manuals/ because we do not run 
    ## `R CMD check`.
    manuals_dir = "../manuals"
    if BBScorevars.is_workflow:
        pass
    elif (source_node):
        print "BBS> [stage6] mkdir %s" % manuals_dir
        os.mkdir(manuals_dir)
    print "BBS> [stage6] BEGIN copying outgoing packages from %s." % fresh_pkgs_subdir
    pkgType = BBScorevars.getNodeSpec(node_hostname, 'pkgType')
    meat_index_file = os.path.join(BBScorevars.Central_rdir.path, BBScorevars.meat_index_file)
    dcf = open(meat_index_file, 'r')
    pkgs = bbs.parse.readPkgsFromDCF(dcf, node_id, pkgType)
    dcf.close()
    for pkg in pkgs:
        if pkgMustBeRejected(node_hostname, node_id, pkg):
            continue
        dcf = open(meat_index_file, 'r')
        version = bbs.parse.getPkgFieldFromDCF(dcf, pkg, 'Version', BBScorevars.meat_index_file)
        dcf.close()
        ## Copy pkg from 'fresh_pkgs_subdir2'.
        pkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkg_file = os.path.join(fresh_pkgs_subdir, pkg_file)
        print "BBS> [stage6]   - copying %s to OUTGOING folder ..." % pkg_file
        if os.path.exists(pkg_file):
            shutil.copy(pkg_file, ".")
        else:
            print "BBS> [stage6]     SKIPPED (file %s doesn't exist)" % pkg_file
        ## Get reference manual from pkg.Rcheck directory.
        if BBScorevars.is_workflow:
            pass
        elif source_node:
            pdf_file = "%s/meat/%s.Rcheck/%s-manual.pdf" % \
                       (BBScorevars.getenv('BBS_WORK_TOPDIR'), pkg, pkg)
            print "BBS> [stage6]   - copying %s manual to OUTGOING/manuals folder..." % pkg
            if os.path.exists(pdf_file):
                shutil.copy(pdf_file, "%s/%s.pdf" % (manuals_dir, pkg))
            else:
                print "BBS> [stage6]     SKIPPED (file %s doesn't exist)" % pdf_file
    print "BBS> [stage6] END copying outgoing packages from %s." % fresh_pkgs_subdir
    return

def extractTargetPkgListFromMeatIndex():
    pkgType = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgType')
    Central_rdir = BBScorevars.Central_rdir
    dcf = Central_rdir.WOpen(BBScorevars.meat_index_file)
    target_pkgs = bbs.parse.readPkgsFromDCF(dcf, BBSvars.node_hostname, pkgType)
    dcf.close()
    return target_pkgs

def stage6_make_OUTGOING():
    ## Create working directory
    OUTGOING_dir = os.path.join(BBScorevars.Central_rdir.path, "OUTGOING")
    print "BBS> [stage6] remake_dir %s" % OUTGOING_dir
    bbs.fileutils.remake_dir(OUTGOING_dir)
    ## Loop over each element of the OUTGOING map
    OUTGOING_map = BBScorevars.getenv('BBS_OUTGOING_MAP')
    map_elts = OUTGOING_map.split(" ")
    for map_elt in map_elts:
        tmp = map_elt.split(":")
        if len(tmp) != 2:
            sys.exit("ERROR: Invalid OUTGOING map element %s" % map_elt)
        source_node = False
        if tmp[0] == "source":
            source_node = True
        OUTGOING_subdir = os.path.join(OUTGOING_dir, tmp[0])
        print "BBS> [stage6] mkdir %s" % OUTGOING_subdir
        os.mkdir(OUTGOING_subdir)
        print "BBS> [stage6] cd %s/" % OUTGOING_subdir
        os.chdir(OUTGOING_subdir)
        tmp2 = tmp[1].split("+")
        if len(tmp2) == 1:
            copy_outgoing_pkgs(tmp[1], source_node)
        elif len(tmp2) == 2:
            make_outgoing_biarch_pkgs(tmp2[0], tmp2[1])
        else:
            sys.exit("ERROR: Invalid OUTGOING map element %s" % map_elt)
    return

def get_node_info():
    logging.info("Node Info:")
    r_version = get_r_version()
    osys = BBScorevars.getNodeSpec(BUILDER_ID, "OS")
    arch = BBScorevars.getNodeSpec(BUILDER_ID, "Arch")
    plat = BBScorevars.getNodeSpec(BUILDER_ID, "Platform")
    send_message({
        "status": "node_info",
        "r_version": r_version,
        "os": osys,
        "arch": arch,
        "platform": plat,
        "body": "node_info",
        "bioc_version": ENVIR['bbs_Bioc_version']})
    logging.info("\n os: " + osys +  "\n r_version: " + r_version +
                 "\n bioc_version: " + ENVIR['bbs_Bioc_version'])

def STAGE5_loop(srcpkg_paths, nb_cpu):
    total = len(srcpkg_paths)
    job_queue = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.buildbin'
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.buildbin_rdir)
        job_queue.append(job)
    nb_jobs = len(job_queue)
    print "BBS> BEGIN STAGE5 loop."
    t0 = time.time()
    nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
                                           BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE5 loop."
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE5 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d srcpkg file(s) in working dir" % total
    print "BBS>   o %d srcpkg file(s) queued and processed" % nb_jobs
    print "BBS>   o %d binpkg file(s) produced" % nb_products
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return

def get_pkgs(dcf, node_hostname=None, node_id=None):
    if node_hostname:
        pkgType = BBScorevars.getNodeSpec(node_hostname, 'pkgType')
        pkgs = bbs.parse.readPkgsFromDCF(dcf, node_id, pkgType)
    else:
        pkgs = bbs.parse.readPkgsFromDCF(dcf)
        pkgs.sort(lambda x, y: cmp(string.lower(x), string.lower(y)))
    return pkgs

def set_NODES(fancynames_in_one_string):
    fancynames = fancynames_in_one_string.split(' ')
    for fancyname in fancynames:
        if fancyname == "":
            continue
        id = fancyname.split(":")[0]
        hostname = id.split("-")[0]
        os_html = BBScorevars.getNodeSpec(hostname, 'OS').replace(' ', ' ')
        arch = BBScorevars.getNodeSpec(hostname, 'Arch')
        platform = BBScorevars.getNodeSpec(hostname, 'Platform')
        buildbin = fancyname_has_a_bin_suffix(fancyname)
        pkgs = get_pkgs_from_meat_index(hostname, id)
        node = Node(hostname, id, os_html, arch, platform, buildbin, pkgs)
        NODES.append(node)
    if len(NODES) == 0:
        sys.exit("nothing to report (no nodes) => EXIT.")
    return

def _BiocGreaterThanOrEqualTo(x, y):
    bioc_version = BBScorevars.getenv('BBS_BIOC_VERSION', False)
    # If BBS_BIOC_VERSION is not defined, then we assume it's the latest version.
    if bioc_version == None:
        return True
    parts = bioc_version.split('.')
    x0 = int(parts[0])
    y0 = int(parts[1])
    return x0 > x or (x0 == x and y0 >= y)

def _getMaintainerFromDir(pkg_dir):
    desc_file = getDescFile(pkg_dir)
    FNULL = open(os.devnull, 'w')
    r_home = BBScorevars.getenv('BBS_R_HOME')
    Rscript_cmd = os.path.join(r_home, "bin", "Rscript")
    script_path = os.path.join(BBScorevars.BBS_home, "utils", "getMaintainer.R")
    cmd = [Rscript_cmd, '--vanilla', script_path, desc_file]
    maintainer = subprocess.check_output(cmd, stderr=FNULL)
    if maintainer == 'NA':
        raise DcfFieldNotFoundError(desc_file, 'Maintainer')
    return maintainer

def getSTAGE2cmd(pkg, version):
    cmd = '%s CMD INSTALL %s' % (BBSvars.r_cmd, pkg)
    if sys.platform == "win32":
        win_archs = _supportedWinArchs(pkg)
        if _mustRunSTAGE2InMultiarchMode() and len(win_archs) == 2:
            curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
            srcpkg_file = pkg + '_' + version + '.tar.gz'
            srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
                         srcpkg_file
            cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
                  '%s CMD INSTALL --merge-multiarch %s' % \
                  (BBSvars.r_cmd, srcpkg_file) + ' && ' + \
                  'rm %s' % srcpkg_file
        else:
            cmd = '%s --arch %s CMD INSTALL --no-multiarch %s' % \
                  (BBSvars.r_cmd, win_archs[0], pkg)
    return cmd

def prepare_STAGE5_job_queue(srcpkg_paths):
    print "BBS> Preparing STAGE5 job queue ... ",
    stage = 'buildbin'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.buildbin_rdir)
        jobs.append(job)
    print "OK"
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue

def getSTAGE2cmd(pkg, version):
    prepend = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend')
    if sys.platform != "win32":
        cmd = '%s %s' % (_get_RINSTALL_cmd0(), pkg)
    else:
        prepend_win = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend.win')
        if prepend_win != None:
            prepend = prepend_win
        win_archs = _supportedWinArchs(pkg)
        if _mustRunSTAGE2InMultiarchMode() and len(win_archs) >= 2:
            ## Here is what Dan's commit message says about why BBS uses this
            ## very long and complicated compound command to install packages
            ## in multiarch mode on Windows during STAGE2 (see
            ## 'git show 87822fb346e04b4301d0c2efd7ec1a2a8762e93a'):
            ##   Install STAGE2 target pkgs to zip+libdir first, then install
            ##   zip. This mitigates the problem where the INSTALL times out
            ##   after installing only one architecture. Now, if the install
            ##   times out, it has only failed to install a package to a
            ##   temporary libdir, and the previous installation is still
            ##   intact (or pkg is not installed at all), so dependent packages
            ##   will not complain about the timed-out package being only
            ##   available for one architecture.
            ##   We still need to figure out why mzR in particular times out
            ##   during INSTALL. When run manually (admittedly not during
            ##   peak load times) the install takes ~ 5-6 minutes, even when
            ##   done via the build system.
            curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
            srcpkg_file = pkg + '_' + version + '.tar.gz'
            srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
                         srcpkg_file
            zip_file = srcpkg_file.replace(".tar.gz", ".zip")
            cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
                  _get_BuildBinPkg_cmd(srcpkg_file, win_archs) + ' && ' + \
                  '%s %s' % (_get_RINSTALL_cmd0(), zip_file) + ' && ' + \
                  'rm %s %s' % (srcpkg_file, zip_file)
        else:
            cmd = '%s %s' % (_get_RINSTALL_cmd0(win_archs), pkg)
    if prepend != None:
        cmd = '%s %s' % (prepend, cmd)
    return cmd

def getSTAGE2cmd(pkg, version):
    cmd = '%s CMD INSTALL %s' % (BBSvars.r_cmd, pkg)
    if sys.platform == "win32":
        win_archs = _supportedWinArchs(pkg)
        if _mustRunSTAGE2InMultiarchMode() and len(win_archs) == 2:
            curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
            srcpkg_file = pkg + '_' + version + '.tar.gz'
            srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
                         srcpkg_file
            zipfile = srcpkg_file.replace(".tar.gz", ".zip")
            cmd = 'rm -rf %s.buildbin-libdir' % pkg + ' && ' + \
                  'mkdir %s.buildbin-libdir ' % pkg + ' && ' + \
                  '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
                  '%s CMD INSTALL --build --library=%s.buildbin-libdir --merge-multiarch %s' % \
                  (BBSvars.r_cmd, pkg, srcpkg_file) + ' && ' + \
                  '%s CMD INSTALL %s ' % (BBSvars.r_cmd,
                    zipfile)  + ' && ' + \
                  'rm %s %s' % (srcpkg_file, zipfile)
        else:
            cmd = '%s --arch %s CMD INSTALL --no-multiarch %s' % \
                  (BBSvars.r_cmd, win_archs[0], pkg)
    return cmd