Python BBScorevars.getNodeSpec方法代码示例

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def get_node_info():
    r_version = get_r_version()
    osys = BBScorevars.getNodeSpec(builder_id, "OS")
    arch = BBScorevars.getNodeSpec(builder_id, "Arch")
    plat = BBScorevars.getNodeSpec(builder_id, "Platform")
    send_message({"status": "node_info", "r_version": r_version,
        "os": osys, "arch": arch, "platform": plat, "body": "node_info",
        "bioc_version": os.getenv("BBS_BIOC_VERSION")})

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def update_packages_file():
    global repos
    
    
    r_version = bioc_r_map[os.getenv("BBS_BIOC_VERSION")]
    if (platform.system() == "Darwin"):
        pkg_type = BBScorevars.getNodeSpec(builder_id, "pkgType")
        if pkg_type == "mac.binary.leopard":
            os_seg = "bin/macosx/contrib/%s" % r_version
        else:
            os_seg = "bin/macosx/mavericks/contrib/%s" % r_version
    elif (platform.system() == "Linux"):
        os_seg = "src/contrib"
    else:
        os_seg = "bin/windows/contrib/%s" % r_version
    
    
    if (manifest['repository'] == 'course'):
        repos = "/loc/www/bioconductor-test.fhcrc.org/course-packages/%s" % os_seg
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/","http://bioconductor.org/")
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/course-packages"
    elif (manifest['repository'] == 'scratch'):
        repos = '/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s/%s' % (manifest['bioc_version'], os_seg)
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/scratch-repos/","http://bioconductor.org/scratch-repos/")
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s" % manifest['bioc_version']
    
    pkg_type = BBScorevars.getNodeSpec(builder_id, "pkgType")
    if pkg_type == "mac.binary.leopard":
        pkg_type = "mac.binary"
    command = \
        "%s [email protected] 'R -f %s/update-repo.R --args %s %s'" \
        % (packagebuilder_ssh_cmd, script_loc, repos, pkg_type)
    print("update packages command: ")
    print(command)
    retcode = subprocess.call(command, shell=True)
    print "retcode for update packages: %d" % retcode
    send_message({"status": "post_processing", "retcode": retcode, "body": "Updated packages list"})
    if retcode != 0:
        send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Updating packages failed.",
            "build_product": build_product, "url": url})
        sys.exit("Updating packages failed")
    if (manifest['repository'] == 'course' or manifest['repository'] == 'scratch'):
        command = "%s [email protected] \"source ~/.bash_profile && cd /home/biocadmin/bioc-test-web/bioconductor.org && rake deploy_production\"" % \
            packagebuilder_ssh_cmd
        print("sync command = ")
        print(command)
        retcode = subprocess.call(command, shell=True)
        send_message({"status": "post_processing", "retcode": retcode, "body": "Synced repository to website",
            "build_product": build_product, "url": url})
        if retcode != 0:
            send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Syncing repository failed",
                "build_product": build_product, "url": url})
            sys.exit("Sync to website failed")
        send_message({"status": "post_processing_complete", "retcode": retcode, "body": "Post-processing complete.",
            "build_product": build_product, "url": url})

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def copy_outgoing_pkgs(fresh_pkgs_subdir, source_node):
    tmp = fresh_pkgs_subdir.split("/")
    if len(tmp) != 2:
        sys.exit("ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)" % fresh_pkgs_subdir)
    node_id = tmp[0]
    node_hostname = node_id.split("-")[0]
    fileext = BBScorevars.getNodeSpec(node_hostname, 'pkgFileExt')
    fresh_pkgs_subdir = os.path.join(BBScorevars.nodes_rdir.path, fresh_pkgs_subdir)

    ## Workflow packages do not have manuals/ because we do not run 
    ## `R CMD check`.
    manuals_dir = "../manuals"
    if BBScorevars.is_workflow:
        pass
    elif (source_node):
        print "BBS> [stage6] mkdir %s" % manuals_dir
        os.mkdir(manuals_dir)
    print "BBS> [stage6] BEGIN copying outgoing packages from %s." % fresh_pkgs_subdir
    pkgType = BBScorevars.getNodeSpec(node_hostname, 'pkgType')
    meat_index_file = os.path.join(BBScorevars.Central_rdir.path, BBScorevars.meat_index_file)
    dcf = open(meat_index_file, 'r')
    pkgs = bbs.parse.readPkgsFromDCF(dcf, node_id, pkgType)
    dcf.close()
    for pkg in pkgs:
        if pkgMustBeRejected(node_hostname, node_id, pkg):
            continue
        dcf = open(meat_index_file, 'r')
        version = bbs.parse.getPkgFieldFromDCF(dcf, pkg, 'Version', BBScorevars.meat_index_file)
        dcf.close()
        ## Copy pkg from 'fresh_pkgs_subdir2'.
        pkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkg_file = os.path.join(fresh_pkgs_subdir, pkg_file)
        print "BBS> [stage6]   - copying %s to OUTGOING folder ..." % pkg_file
        if os.path.exists(pkg_file):
            shutil.copy(pkg_file, ".")
        else:
            print "BBS> [stage6]     SKIPPED (file %s doesn't exist)" % pkg_file
        ## Get reference manual from pkg.Rcheck directory.
        if BBScorevars.is_workflow:
            pass
        elif source_node:
            pdf_file = "%s/meat/%s.Rcheck/%s-manual.pdf" % \
                       (BBScorevars.getenv('BBS_WORK_TOPDIR'), pkg, pkg)
            print "BBS> [stage6]   - copying %s manual to OUTGOING/manuals folder..." % pkg
            if os.path.exists(pdf_file):
                shutil.copy(pdf_file, "%s/%s.pdf" % (manuals_dir, pkg))
            else:
                print "BBS> [stage6]     SKIPPED (file %s doesn't exist)" % pdf_file
    print "BBS> [stage6] END copying outgoing packages from %s." % fresh_pkgs_subdir
    return

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def get_node_info():
    logging.info("Node Info:")
    r_version = get_r_version()
    osys = BBScorevars.getNodeSpec(BUILDER_ID, "OS")
    arch = BBScorevars.getNodeSpec(BUILDER_ID, "Arch")
    plat = BBScorevars.getNodeSpec(BUILDER_ID, "Platform")
    send_message({
        "status": "node_info",
        "r_version": r_version,
        "os": osys,
        "arch": arch,
        "platform": plat,
        "body": "node_info",
        "bioc_version": ENVIR['bbs_Bioc_version']})
    logging.info("\n os: " + osys +  "\n r_version: " + r_version +
                 "\n bioc_version: " + ENVIR['bbs_Bioc_version'])

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def extractTargetPkgListFromMeatIndex():
    pkgType = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgType')
    Central_rdir = BBScorevars.Central_rdir
    dcf = Central_rdir.WOpen(BBScorevars.meat_index_file)
    target_pkgs = bbs.parse.readPkgsFromDCF(dcf, BBSvars.node_hostname, pkgType)
    dcf.close()
    return target_pkgs

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def STAGE5_loop(srcpkg_paths, nb_cpu):
    total = len(srcpkg_paths)
    job_queue = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.buildbin'
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.buildbin_rdir)
        job_queue.append(job)
    nb_jobs = len(job_queue)
    print "BBS> BEGIN STAGE5 loop."
    t0 = time.time()
    nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
                                           BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE5 loop."
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE5 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d srcpkg file(s) in working dir" % total
    print "BBS>   o %d srcpkg file(s) queued and processed" % nb_jobs
    print "BBS>   o %d binpkg file(s) produced" % nb_products
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def get_pkgs(dcf, node_hostname=None, node_id=None):
    if node_hostname:
        pkgType = BBScorevars.getNodeSpec(node_hostname, 'pkgType')
        pkgs = bbs.parse.readPkgsFromDCF(dcf, node_id, pkgType)
    else:
        pkgs = bbs.parse.readPkgsFromDCF(dcf)
        pkgs.sort(lambda x, y: cmp(string.lower(x), string.lower(y)))
    return pkgs

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def set_NODES(fancynames_in_one_string):
    fancynames = fancynames_in_one_string.split(' ')
    for fancyname in fancynames:
        if fancyname == "":
            continue
        id = fancyname.split(":")[0]
        hostname = id.split("-")[0]
        os_html = BBScorevars.getNodeSpec(hostname, 'OS').replace(' ', ' ')
        arch = BBScorevars.getNodeSpec(hostname, 'Arch')
        platform = BBScorevars.getNodeSpec(hostname, 'Platform')
        buildbin = fancyname_has_a_bin_suffix(fancyname)
        pkgs = get_pkgs_from_meat_index(hostname, id)
        node = Node(hostname, id, os_html, arch, platform, buildbin, pkgs)
        NODES.append(node)
    if len(NODES) == 0:
        sys.exit("nothing to report (no nodes) => EXIT.")
    return

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def prepare_STAGE5_job_queue(srcpkg_paths):
    print "BBS> Preparing STAGE5 job queue ... ",
    stage = 'buildbin'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.buildbin_rdir)
        jobs.append(job)
    print "OK"
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def propagate_package():
    global build_product
    global repos
    global url
    pkg_type = BBScorevars.getNodeSpec(builder_id, "pkgType")
    ext = BBScorevars.pkgType2FileExt[pkg_type]
    files = os.listdir(working_dir)
    build_product = filter(lambda x: x.endswith(ext), files)[0]
    
    
    # now install the package
    #r_cmd = "%s CMD INSTALL %s" % (os.getenv("BBS_R_CMD"), build_product)
    
    #send_message({"body": "Installing package", "status": "post_processing", "retcode": 0})
    
    #retcode = subprocess.call(r_cmd, shell=True)
    
    #send_message({"body": "Installed package", "status": "post_processing", "retcode": retcode})
    
    #if retcode != 0:
    #    sys.exit("package install failed")
    
    r_version = bioc_r_map[os.getenv("BBS_BIOC_VERSION")]    
    if (platform.system() == "Darwin"):
        os_seg = "bin/macosx/mavericks/contrib/%s" % r_version
    elif (platform.system() == "Linux"):
        os_seg = "src/contrib"
    else:
        os_seg = "bin/windows/contrib/%s" % r_version
    
    if (manifest['repository'] == 'course'):
        repos = "/loc/www/bioconductor-test.fhcrc.org/course-packages/%s" % os_seg
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/","http://bioconductor.org/")
    elif (manifest['repository'] == 'scratch'):
        repos = '/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s/%s' % (manifest['bioc_version'], os_seg)
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/scratch-repos/","http://bioconductor.org/scratch-repos/")
    
    
    
    url += "/" + build_product
    
    rawsize = os.path.getsize(build_product)
    kib = rawsize / float(1024)
    filesize = "%.2f" % kib
    
    files_to_delete = "%s/%s_*.%s" % (repos, package_name, ext)
    
    if (platform.system() == "Windows"):
        retcode = subprocess.call("c:/cygwin/bin/ssh.exe -qi %s/.packagebuilder.private_key.rsa -o StrictHostKeyChecking=no [email protected] 'rm -f %s/%s_*.zip'" % (os.environ["PACKAGEBUILDER_HOME"], 
          repos, package_name))
    else:
        retcode = ssh("rm -f %s" % files_to_delete)
    
    print("result of deleting files: %d" % retcode)
    send_message({"body": "Pruning older packages from repository", "status": "post_processing", "retcode": retcode})
    if retcode != 0:
        sys.exit("repos prune failed")
    
    if (platform.system() == "Windows"):
        print("platform.system() == 'Windows', running chmod commands...")
        chmod_retcode = subprocess.call("chmod a+r %s" % os.path.join(working_dir, package_name))
        print("chmod_retcode = %d" % chmod_retcode)
        send_message({"status": "chmod_retcode", "body": \
          "chmod_retcode=%d" % chmod_retcode, "retcode": chmod_retcode})
        command = "c:/cygwin/bin/scp.exe -qi %s/.packagebuilder.private_key.rsa -o StrictHostKeyChecking=no %s [email protected]:%s/" % (os.environ["PACKAGEBUILDER_HOME"],
          build_product, repos)
        print("command = %s" % command)
        retcode = subprocess.call(command)
        remote_chmod_retcode = subprocess.call("c:/cygwin/bin/ssh.exe -qi %s/.packagebuilder.private_key.rsa -o StrictHostKeyChecking=no [email protected] 'chmod a+r %s/%s_*.zip'" % (os.environ["PACKAGEBUILDER_HOME"],
          repos, package_name))
        print("remote_chmod_retcode = %s" % remote_chmod_retcode)
    else:
        print("chmod code not run, because platform.system() == %s" % platform.system())
        retcode = scp(build_product, repos)
    
    print("result of copying file: %d" % retcode)
    send_message({"body": "Copied build file to repository", "status": "post_processing", "retcode": retcode,
        "build_product": build_product, "filesize": filesize})
    if retcode != 0:
        sys.exit("copying file to repository failed")

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def propagate_package():
    global build_product
    global repos
    global url
    pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")
    ext = BBScorevars.pkgType2FileExt[pkg_type]
    files = os.listdir(working_dir)
    build_product = filter(lambda x: x.endswith(ext), files)[0]
    r_version = BIOC_R_MAP[ENVIR['bbs_Bioc_version']]
    if (platform.system() == "Darwin"):
        os_seg = "bin/macosx/mavericks/contrib/%s" % r_version
    elif (platform.system() == "Linux"):
        os_seg = "src/contrib"
    else:
        os_seg = "bin/windows/contrib/%s" % r_version

    if (manifest['repository'] == 'course'):
        repos = "/loc/www/bioconductor-test.fhcrc.org/course-packages/%s" % os_seg
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/",
            HOSTS['bioc'])
    elif (manifest['repository'] == 'scratch'):
        repos = '/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s/%s' % (
            manifest['bioc_version'], os_seg)
        url = repos.replace(
            "/loc/www/bioconductor-test.fhcrc.org/scratch-repos/",
            HOSTS['bioc'] + '/scratch-repos')

    url += "/" + build_product

    rawsize = os.path.getsize(build_product)
    kib = rawsize / float(1024)
    filesize = "%.2f" % kib

    package_name = manifest['job_id'].split("_")[0]
    files_to_delete = "%s/%s_*.%s" % (repos, package_name, ext)

    if (platform.system() == "Windows"):
        command = "c:/cygwin/bin/ssh.exe -qi %s -o StrictHostKeyChecking=no [email protected]%s 'rm -f %s/%s_*.zip'"
        command = command % (ENVIR['spb_RSA_key'], ENVIR['spb_staging_url'], repos, package_name)
        retcode = subprocess.call(command)
    else:
        logging.info("propagate_package() files_to_delete: %s" %
                      files_to_delete)
        retcode = ssh("rm -f %s" % files_to_delete)

    logging.info("Finished propagate_package().\n Result of deleting files: %d." % retcode)
    send_message({
        "body": "Pruning older packages from repository. ",
        "status": "post_processing",
        "retcode": retcode
    })
    if retcode != 0:
        logging.error("propagate_package() Failed to prune repos.")
        sys.exit("repos prune failed")

    if (platform.system() == "Windows"):
        logging.info("propagate_package() Windows chmod")
        chmod_retcode = subprocess.call(
            "chmod a+r %s" % os.path.join(working_dir, package_name))
        logging.info("propagate_package() Windows chmod_retcode = %d" %
                      chmod_retcode)
        send_message({
            "status": "chmod_retcode",
            "body": "chmod_retcode=%d" % chmod_retcode,
            "retcode": chmod_retcode
        })
        command = "c:/cygwin/bin/scp.exe -qi %s -o StrictHostKeyChecking=no %s [email protected]%s:%s/"
        command = command % (ENVIR['spb_RSA_key'], build_product, ENVIR['spb_staging_url'], repos)
        logging.info("propagate_package() Windows scp command = %s." %
                      command)
        retcode = subprocess.call(command)
        command = "c:/cygwin/bin/ssh.exe -qi %s -o StrictHostKeyChecking=no [email protected]%s 'chmod a+r %s/%s_*.zip'"
        command = command % (ENVIR['spb_RSA_key'], ENVIR['spb_staging_url'], repos, package_name)
        remote_chmod_retcode = subprocess.call(command)
        logging.info("propagate_package() Windows remote_chmod_retcode = %s" %
                      remote_chmod_retcode)
    else:
        logging.info("propagate_package() %s chmod not run" %
                      platform.system())
        retcode = scp(build_product, repos)

    logging.info("propagate_package() Result of copying file: %d" % retcode)
    send_message({
        "body": "Copied build file to repository. ",
        "status": "post_processing",
        "retcode": retcode,
        "build_product": build_product,
        "filesize": filesize
    })
    if retcode != 0:
        logging.error("propagate_package() Failed to copy file to repository.")
        sys.exit("failed to copy file to repository")

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def build_package(source_build):
    global pkg_type

    pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")

    buildmsg = None
    if (source_build):
        buildmsg = "building"
    else:
        buildmsg = "buildingbin"

    if ((not source_build) and (pkg_type == "source")):
        send_message({"status": "skip_buildbin", "body": "skipped"})
        logging.info("Skip buildbin")
        return(0)

    if (not source_build):
        if platform.system() == "Darwin":
            pkg_type = "mac.binary.mavericks"
        elif platform.system() == "Linux":
            pkg_type = "source"
        elif platform.system() == "Windows":
            pkg_type = "win.binary"
        else:
            pkg_type = "source"
        send_message({"status": "starting_buildbin", "body": ""})
        logging.info("Start buildbin")

    global message_sequence
    global warnings
    message_sequence = 1
    flags = "--keep-empty-dirs --no-resave-data"

    if (source_build):
        package_name = manifest['job_id'].split("_")[0]
        r_cmd = "%s CMD build %s %s" % \
                (ENVIR['bbs_R_cmd'], flags, package_name)
    else:
        if pkg_type == "mac.binary" or pkg_type == "mac.binary.mavericks":
            libdir = "libdir"
            if os.path.exists(libdir):
                _call("rm -rf %s" % libdir, False)
            if (not (os.path.exists(libdir))):
                os.mkdir(libdir)
            r_cmd = os.environ['SPB_HOME'] + "/build-universal.sh %s %s" % (
                get_source_tarball_name(), libdir)

    status = None
    if (source_build):
        status = "r_cmd"
        outfile = "R.out"
    else:
        status = "r_buildbin_cmd"
        outfile = "Rbuildbin.out"
    logging.debug("Before build, working dir is %s." %
                  working_dir)

    start_time = datetime.datetime.now()
    if ((not source_build) and pkg_type == "win.binary"):
        retcode = win_multiarch_buildbin(buildmsg)
    else:
        send_message({"status": status, "body": r_cmd})
        retcode = do_build(r_cmd, buildmsg, source_build)

    stop_time = datetime.datetime.now()
    time_dif = stop_time - start_time
    min_time, sec_time = divmod(time_dif.seconds,60)
    sec_time = str(format(float(str(time_dif).split(":")[2]), '.2f'))
    elapsed_time = str(min_time) + " minutes " + sec_time + " seconds"

    # check for warnings
    out_fh = open(outfile)
    warnings = False
    for line in out_fh:
        if line.lower().startswith("warning:"):
            warnings = True
        if line.lower().startswith("error:"):
            retcode = 1
    out_fh.close()

    complete_status = None
    if (source_build):
        complete_status = "build_complete"
    else:
        complete_status = "buildbin_complete"

    send_message({
        "status": complete_status,
        "retcode": retcode,
        "warnings": warnings,
        "body": "Build completed with status %d" % retcode,
        "elapsed_time": elapsed_time})
    logging.info(complete_status + "\n Build completed with status: " +
                 str(retcode) + " Elapsed time: " + elapsed_time)

    return (retcode)

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def is_build_required(manifest):
    global svn_url_global
    svn_url_global = manifest['svn_url']
    package_name = manifest['job_id'].split("_")[0]
    logging.info("Starting is_build_required() '%s'." % package_name)

    if (get_package_source() == "svn"):
        description_url = manifest['svn_url'].rstrip("/") + "/DESCRIPTION"
        logging.debug("is_build_required() package source is svn" +
            "\n  description_url = " + description_url +
            "\n  svn_user ="  + ENVIR['svn_user'] +
            "\n  svn_pass = " + ENVIR['svn_pass'])
        try:
            description = subprocess.Popen([
                "curl", "-k", "-s", "--user", "%s:%s" %
                (ENVIR['svn_user'], ENVIR['svn_pass']),
                description_url
            ], stdout=subprocess.PIPE).communicate()[0]
            # TODO - handle it if description does not exist
        except:
            logging.error("is_build_required() curl exception: %s.",
                          sys.exc_info()[0])
            raise

        logging.debug("is_build_required()" +
                      "\n  description = %s" % description +
                      "\n  length = %d" % len(description))

        dcf_file = dcf.DcfRecordParser(description.rstrip().split("\n"))
        send_dcf_info(dcf_file)

        svn_version = dcf_file.getValue("Version")
    elif get_package_source() == "github":
        github_url = re.sub(r'\.git$', '', manifest['svn_url'])
        if not github_url.endswith("/"):
            github_url += "/"
        # We only build the master branch. There had better be one.
        # (technically we build whatever the default branch is, but
        # this step looks at master because to find out what the
        # default branch is at this point we would need octokit here).
        github_url += "master/DESCRIPTION"
        github_url = github_url.replace("https://github.com",
          "https://raw.githubusercontent.com")
        f = urllib2.urlopen(github_url)
        dcf_text = f.read()
        dcf_file = dcf.DcfRecordParser(dcf_text.rstrip().split("\n"))
        send_dcf_info(dcf_file)
        svn_version = dcf_file.getValue("Version")
    elif get_package_source() == "tracker":
        tmp = manifest["svn_url"].split("/")
        pkgname = tmp[len(tmp)-1].replace(".tar.gz", "")
        if (pkgname.find("_") == -1): # package name doesn't have version in it
            return(True) # TODO - download tarball and examine DESCRIPTION file
        svn_version = pkgname.split("_")[1]

    if ("force" in manifest.keys()):
        if (manifest['force'] is True):
            return(True)

    r_version = BIOC_R_MAP[ENVIR['bbs_Bioc_version']]
    pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")

    cran_repo_map = {
        'source': "src/contrib",
        'win.binary': "bin/windows/contrib/" + r_version,
        'win64.binary': "bin/windows64/contrib/" + r_version,
        'mac.binary': "bin/macosx/contrib/" + r_version,
        'mac.binary.mavericks': "bin/macosx/mavericks/contrib/" + r_version
    }
    # todo - put repos url in config file (or get it from user)
    base_repo_url = HOSTS['bioc']
    if (manifest['repository'] == 'course'):
        base_repo_url += '/course-packages'
    elif (manifest['repository'] == 'scratch'):
        base_repo_url += '/scratch_repos/' + manifest['bioc_version']

    repository_url = "%s/%s/PACKAGES" % (base_repo_url, cran_repo_map[pkg_type])
    # What if there is no file at this url?
    packages = subprocess.Popen(["curl", "-k", "-s", repository_url],
        stdout=subprocess.PIPE).communicate()[0]
    inpackage = False
    repository_version = False
    for line in packages.split("\n"):
        if line == "Package: %s" % package_name:
            inpackage = True
        if (line.startswith("Version: ") and inpackage):
            repository_version = line.split(": ")[1]
            break
    if not repository_version:
        return True # package hasn't been pushed to repo before
    logging.debug("is_build_required()" +
                  "\n  [%s] svn version is %s, repository version is %s" %
                  (package_name, svn_version, repository_version))
    return svn_version != repository_version

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def make_outgoing_biarch_pkgs(fresh_pkgs_subdir1, fresh_pkgs_subdir2):
    tmp1 = fresh_pkgs_subdir1.split("/")
    if len(tmp1) != 2:
        sys.exit("ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)" % fresh_pkgs_subdir1)
    node1_id = tmp1[0]
    node1_hostname = node1_id.split("-")[0]
    tmp2 = fresh_pkgs_subdir2.split("/")
    if len(tmp2) != 2:
        sys.exit("ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)" % fresh_pkgs_subdir2)
    node2_id = tmp2[0]
    node2_hostname = node2_id.split("-")[0]
    ## Check that node1 and node2 are registered as Windows i386 and x64
    ## builders, respectively
    pkgType1 = BBScorevars.getNodeSpec(node1_hostname, 'pkgType')
    if pkgType1 != "win.binary":
        sys.exit("ERROR: %s pkgType is not \"win.binary\"" % node1_hostname)
    pkgType2 = BBScorevars.getNodeSpec(node2_hostname, 'pkgType')
    if pkgType2 != "win64.binary":
        sys.exit("ERROR: %s pkgType is not \"win64.binary\"" % node2_hostname)
    fileext = BBScorevars.getNodeSpec(node1_hostname, 'pkgFileExt')
    fileext2 = BBScorevars.getNodeSpec(node2_hostname, 'pkgFileExt')
    if fileext2 != fileext:
        sys.exit("ERROR: %s pkgFileExt and %s pkgFileExt differ" % (node1_hostname, node2_hostname))
    fresh_pkgs_subdir1 = os.path.join(BBScorevars.nodes_rdir.path, fresh_pkgs_subdir1)
    fresh_pkgs_subdir2 = os.path.join(BBScorevars.nodes_rdir.path, fresh_pkgs_subdir2)
    print "BBS> [stage6] BEGIN making outgoing bi-arch packages from %s and %s." % (fresh_pkgs_subdir1, fresh_pkgs_subdir2)
    ## Get lists of supported pkgs for node1 and node2
    meat_index_file = os.path.join(BBScorevars.Central_rdir.path, BBScorevars.meat_index_file)
    dcf = open(meat_index_file, 'r')
    pkgs1 = bbs.parse.readPkgsFromDCF(dcf, node1_id, pkgType1)
    dcf.close()
    dcf = open(meat_index_file, 'r')
    pkgs2 = bbs.parse.readPkgsFromDCF(dcf, node2_id, pkgType2)
    dcf.close()
    ## Loop on list of supported pkgs
    pkgs0 = set(pkgs1 + pkgs2)
    nb_products = 0
    t1 = time.time()
    for pkg in pkgs0:
        dcf = open(meat_index_file, 'r')
        version = bbs.parse.getPkgFieldFromDCF(dcf, pkg, 'Version', BBScorevars.meat_index_file)
        dcf.close()
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        if pkg not in pkgs1:
            if pkgMustBeRejected(node2_hostname, node2_id, pkg):
                continue
            ## Copy pkg from 'fresh_pkgs_subdir2'
            binpkg_file2 = os.path.join(fresh_pkgs_subdir2, binpkg_file)
            shutil.copy(binpkg_file2, ".")
            nb_products += 1
            continue
        if pkg not in pkgs2:
            if pkgMustBeRejected(node1_hostname, node1_id, pkg):
                continue
            ## Copy pkg from 'fresh_pkgs_subdir1'
            binpkg_file1 = os.path.join(fresh_pkgs_subdir1, binpkg_file)
            shutil.copy(binpkg_file1, ".")
            nb_products += 1
            continue
        if pkgMustBeRejected(node1_hostname, node1_id, pkg) or pkgMustBeRejected(node2_hostname, node2_id, pkg):
            continue
        ## Merge
        syscmd = '%s/utils/merge-win-bin-pkgs.sh %s %s %s %s cleanup' % (BBScorevars.BBS_home, pkg, version, fresh_pkgs_subdir1, fresh_pkgs_subdir2)
        bbs.jobs.doOrDie(syscmd)
        nb_products += 1
    dt = time.time() - t1
    print "BBS> [stage6] END making outgoing bi-arch packages from %s and %s." % (fresh_pkgs_subdir1, fresh_pkgs_subdir2)
    print "BBS> -------------------------------------------------------------"
    print "BBS> [stage6] MERGE(%s, %s) SUMMARY:" % (node1_id, node2_id)
    print "BBS>     o Working dir: %s" % os.getcwd()
    print "BBS>     o %d pkg(s) supported on Windows" % len(pkgs0)
    print "BBS>     o %d binpkg file(s) produced" % nb_products
    print "BBS>     o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return

# 需要导入模块: import BBScorevars [as 别名]
# 或者: from BBScorevars import getNodeSpec [as 别名]
def update_packages_file():
    global repos
    global build_product

    r_version = BIOC_R_MAP[ENVIR['bbs_Bioc_version']]
    if (platform.system() == "Darwin"):
        pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")
        if pkg_type == "mac.binary.leopard":
            os_seg = "bin/macosx/contrib/%s" % r_version
        else:
            os_seg = "bin/macosx/mavericks/contrib/%s" % r_version
    elif (platform.system() == "Linux"):
        os_seg = "src/contrib"
    else:
        os_seg = "bin/windows/contrib/%s" % r_version

    if (manifest['repository'] == 'course'):
        repos = "/loc/www/bioconductor-test.fhcrc.org/course-packages/%s" % os_seg
        url = repos.replace("/loc/www/bioconductor-test.fhcrc.org/",
                            HOSTS['bioc'] + '/')
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/course-packages"
    elif (manifest['repository'] == 'scratch'):
        repos = '/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s/%s' % (
            manifest['bioc_version'], os_seg)
        url = repos.replace(
            "/loc/www/bioconductor-test.fhcrc.org/scratch-repos/",
            HOSTS['bioc'] + "/scratch-repos/")
        script_loc = "/loc/www/bioconductor-test.fhcrc.org/scratch-repos/%s" % manifest['bioc_version']

    pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")
    if pkg_type == "mac.binary.leopard":
        pkg_type = "mac.binary"
    command = "%s [email protected]%s 'R -f %s/update-repo.R --args %s %s'"
    command = command % (packagebuilder_ssh_cmd, ENVIR['spb_staging_url'], script_loc, repos, pkg_type)
    logging.info("update_packages_file() command: %s"  % command)
    retcode = subprocess.call(command, shell=True)
    logging.info("update_packages_file() retcode: %d" % retcode)
    send_message({
        "status": "post_processing",
        "retcode": retcode,
        "body": "Updated packages list. "
    })
    if retcode != 0:
        send_message({
            "status": "post_processing_complete",
            "retcode": retcode,
            "body": "Updating packages failed.",
            "build_product": build_product,
            "url": url
        })
        sys.exit("updating packages failed")
    if (manifest['repository'] == 'course' or manifest['repository'] == 'scratch'):
        command = "%s [email protected]%s \"source ~/.bash_profile && cd /home/biocadmin/bioc-test-web/bioconductor.org && rake deploy_production\""
        command = command % (packagebuilder_ssh_cmd, ENVIR['spb_staging_url'])
        logging.info("update_packages_file() sync command = %s" % command)
        retcode = subprocess.call(command, shell=True)
        send_message({
            "status": "post_processing",
            "retcode": retcode,
            "body": "Synced repository to website. ",
            "build_product": build_product,
            "url": url
        })
        if retcode != 0:
            send_message({
                "status": "post_processing_complete",
                "retcode": retcode,
                "body": "Syncing repository failed",
                "build_product": build_product,
                "url": url})
            sys.exit("sync to website failed")
        send_message({
            "status": "post_processing_complete",
            "retcode": retcode,
            "body": "Post-processing complete.",
            "build_product": build_product,
            "url": url})