当前位置: 首页>>代码示例>>Python>>正文


Python cluster.cluster_view函数代码示例

本文整理汇总了Python中cluster_helper.cluster.cluster_view函数的典型用法代码示例。如果您正苦于以下问题:Python cluster_view函数的具体用法?Python cluster_view怎么用?Python cluster_view使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。


在下文中一共展示了cluster_view函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: create

def create(parallel, dirs, config):
    """Create a cluster based on the provided parallel arguments.

    Returns an IPython view on the cluster, enabling processing on jobs.

    Adds a mincores specification if he have machines with a larger
    number of cores to allow jobs to be batched together for shared
    memory usage.
    """
    profile_dir = utils.safe_makedir(os.path.join(dirs["work"], get_log_dir(config), "ipython"))
    has_mincores = any(x.startswith("mincores=") for x in parallel["resources"])
    if parallel["system_cores"] > 3 and not has_mincores:
        cores = parallel["system_cores"]
        # if we have larger number of cores, leave room for standard batch script and controller
        if parallel["system_cores"] > 30:
            cores = cores - 2
        elif parallel["system_cores"] > 15:
            cores = cores - 1
        parallel["resources"].append("mincores=%s" % cores)
    return ipython_cluster.cluster_view(parallel["scheduler"].lower(), parallel["queue"],
                                        parallel["num_jobs"], parallel["cores_per_job"],
                                        profile=profile_dir, start_wait=parallel["timeout"],
                                        extra_params={"resources": parallel["resources"],
                                                      "mem": parallel["mem"],
                                                      "tag": parallel.get("tag"),
                                                      "run_local": parallel.get("run_local")},
                                        retries=parallel.get("retries"))
开发者ID:SciLifeLab,项目名称:bcbio-nextgen,代码行数:27,代码来源:ipython.py

示例2: create

def create(parallel, dirs, config):
    """Create a cluster based on the provided parallel arguments.

    Returns an IPython view on the cluster, enabling processing on jobs.

    Adds a mincores specification if he have machines with a larger
    number of cores to allow jobs to be batched together for shared
    memory usage.
    """
    profile_dir = utils.safe_makedir(os.path.join(dirs["work"], get_log_dir(config), "ipython"))
    has_mincores = any(x.startswith("mincores=") for x in parallel["resources"])
    cores = min(_get_common_cores(config["resources"]), parallel["system_cores"])
    if cores > 1 and not has_mincores:
        adj_cores = max(1, int(math.floor(cores * float(parallel.get("mem_pct", 1.0)))))
        # if we have less scheduled cores than per machine, use the scheduled count
        if cores > parallel["cores"]:
            cores = parallel["cores"]
        # if we have less total cores required for the entire process, use that
        elif adj_cores > parallel["num_jobs"] * parallel["cores_per_job"]:
            cores = parallel["num_jobs"] * parallel["cores_per_job"]
        else:
            cores = adj_cores
            cores = per_machine_target_cores(cores, parallel["num_jobs"] // cores)
        parallel["resources"].append("mincores=%s" % cores)
    return ipython_cluster.cluster_view(parallel["scheduler"].lower(), parallel["queue"],
                                        parallel["num_jobs"], parallel["cores_per_job"],
                                        profile=profile_dir, start_wait=parallel["timeout"],
                                        extra_params={"resources": parallel["resources"],
                                                      "mem": parallel["mem"],
                                                      "tag": parallel.get("tag"),
                                                      "run_local": parallel.get("run_local")},
                                        retries=parallel.get("retries"))
开发者ID:dh10,项目名称:bcbio-nextgen,代码行数:32,代码来源:ipython.py

示例3: main

def main():
    #files = (sorted(glob.glob("/n/hsphS10/hsphfs2/tanzi_recalled/*alz-*/final/*/*-ready.bam")) +
    #         sorted(glob.glob("/n/hsphS10/hsphfs2/tanzi_recalled/*alz-*/final/*/*-ready.cram")))
    files = (sorted(glob.glob("/n/hsphS10/hsphfs1/chb/projects/tanzi_ad/calls/*alz-*/final/*/*-ready.bam")))
    with cluster_view(scheduler="lsf", queue="hsph_bioinfo", num_jobs=64,
                      cores_per_job=1, extra_params={"mem": "3.3"}) as view:
        view.map(cram_compress, files)
开发者ID:bachu-mahe,项目名称:projects,代码行数:7,代码来源:cram_compress.py

示例4: get_cluster_view

def get_cluster_view(args):
    return ipc.cluster_view(args.scheduler, args.queue,
                          args.num_jobs, args.cores_per_job,
                          start_wait=args.timeout,
                          extra_params={"resources": args.resources,
                                        "mem": args.memory_per_job,
                                        "tag": "singlecell",
                                        "run_local": args.local})
开发者ID:hbc,项目名称:daley_tailseq,代码行数:8,代码来源:cluster.py

示例5: _view_from_parallel

def _view_from_parallel(parallel, work_dir, config):
    """Translate parallel map into options for a cluster view.
    """
    profile_dir = utils.safe_makedir(os.path.join(work_dir, get_log_dir(config), "ipython"))
    return ipython_cluster.cluster_view(parallel["scheduler"].lower(), parallel["queue"],
                                        parallel["num_jobs"], parallel["cores_per_job"],
                                        profile=profile_dir, start_wait=parallel["timeout"],
                                        extra_params={"resources": parallel["resources"]},
                                        retries=parallel.get("retries"))
开发者ID:alneberg,项目名称:bcbio-nextgen,代码行数:9,代码来源:ipython.py

示例6: test_run_analysis

def test_run_analysis():
    tmp_control = idr.tagalign_split(_copy_file_to_testdir(control_file))
    tmp_exp = idr.tagalign_split(_copy_file_to_testdir(exp_file))
    with cluster_view(scheduler, queue, jobs) as view:
        plots, filtered_files = idr.run_analysis(tmp_control, tmp_exp,
                                                 spp_path, idr_runner_path,
                                                 idr_plotter_path, view.map,
                                                 caller)

    all(map(idr.file_exists, filtered_files))
开发者ID:daler,项目名称:spp-idr,代码行数:10,代码来源:idr_test.py

示例7: test_pooled_pseudo_replicate_caller

 def test_pooled_pseudo_replicate_caller(self):
     peak_caller = idr.spp_peak_caller(spp_path)
     control_replicates = idr.tagalign_split(self.control, nfiles=2)
     experimental_replicates = idr.tagalign_split(self.experimental,
                                                  nfiles=2)
     with cluster_view(scheduler, queue, jobs) as view:
         peaks = idr.call_peaks_on_pooled_pseudoreplicates(control_replicates,
                                                         experimental_replicates,
                                                         peak_caller, view.map)
     self.assertTrue(all(map(idr.file_exists, peaks)))
     self.assertTrue(all(map(idr.is_peak_file, peaks)))
开发者ID:daler,项目名称:spp-idr,代码行数:11,代码来源:idr_test.py

示例8: get_cluster_view

def get_cluster_view(args):
    if not os.path.exists("ipython"):
        os.mkdir("ipython")
    return ipc.cluster_view(
        args.scheduler,
        args.queue,
        args.num_jobs,
        args.cores_per_job,
        start_wait=args.timeout,
        profile="ipython",
        extra_params={"resources": args.resources, "mem": args.memory_per_job, "tag": "ts", "run_local": args.local},
    )
开发者ID:hbc,项目名称:daley_tailseq,代码行数:12,代码来源:cluster.py

示例9: get_cluster_view

def get_cluster_view(args, num_jobs=None):
    if not os.path.exists("ipython"):
        os.mkdir("ipython")
        os.mkdir("checkpoint")
    if not num_jobs:
        num_jobs = args.num_jobs
    return ipc.cluster_view(args.scheduler, args.queue,
                          num_jobs, args.cores_per_job,
                          start_wait=args.timeout,
                          profile="ipython",
                          extra_params={"resources": args.resources,
                                        "mem": args.memory_per_job,
                                        "tag": "seqcluster",
                                        "run_local": args.local})
开发者ID:lpantano,项目名称:seqcluster-helper,代码行数:14,代码来源:cluster.py

示例10: create

def create(parallel, dirs, config):
    """Create a cluster based on the provided parallel arguments.

    Returns an IPython view on the cluster, enabling processing on jobs.
    """
    profile_dir = utils.safe_makedir(os.path.join(dirs["work"], get_log_dir(config), "ipython"))
    return ipython_cluster.cluster_view(parallel["scheduler"].lower(), parallel["queue"],
                                        parallel["num_jobs"], parallel["cores_per_job"],
                                        profile=profile_dir, start_wait=parallel["timeout"],
                                        extra_params={"resources": parallel["resources"],
                                                      "mem": parallel["mem"],
                                                      "tag": parallel.get("tag"),
                                                      "run_local": parallel.get("run_local")},
                                        retries=parallel.get("retries"))
开发者ID:torchmed,项目名称:bcbio-nextgen,代码行数:14,代码来源:ipython.py

示例11: runner

def runner(parallel, fn_name, items, work_dir, config):
    """Run a task on an ipython parallel cluster, allowing alternative queue types.

    This will spawn clusters for parallel and custom queue types like multicore
    and high I/O tasks on demand.

    A checkpoint directory keeps track of finished tasks, avoiding spinning up clusters
    for sections that have been previous processed.
    """
    setup_logging(config)
    out = []
    checkpoint_dir = utils.safe_makedir(os.path.join(work_dir, "checkpoints_ipython"))
    checkpoint_file = _get_checkpoint_file(checkpoint_dir, fn_name)
    fn = getattr(__import__("{base}.ipythontasks".format(base=parallel["module"]),
                            fromlist=["ipythontasks"]),
                 fn_name)
    items = [x for x in items if x is not None]
    num_jobs, cores_per_job = find_cores_per_job(fn, parallel, items, config)
    parallel = dictadd(parallel, "cores_per_job", cores_per_job)
    parallel = dictadd(parallel, "num_jobs", num_jobs)
    # already finished, run locally on current machine to collect details
    if os.path.exists(checkpoint_file):
        logger.info("ipython: %s -- local; checkpoint passed" % fn_name)
        for args in items:
            if args:
                data = fn(args)
                if data:
                    out.extend(data)
    # Run on a standard parallel queue
    else:
        logger.info("ipython: %s" % fn_name)
        if len(items) > 0:
            items = [add_cores_to_config(x, cores_per_job) for x in items]
            with ipython_cluster.cluster_view(parallel["scheduler"].lower(), parallel["queue"],
                                              parallel["num_jobs"], parallel["cores_per_job"],
                                              profile=parallel["profile"]) as view:
                for data in view.map_sync(fn, items, track=False):
                    if data:
                        out.extend(data)
    with open(checkpoint_file, "w") as out_handle:
        out_handle.write("done\n")
    return out
开发者ID:joshuashen,项目名称:steady,代码行数:42,代码来源:ipython.py

示例12: find_cores_per_job

                                         ["gene_id"],
                                         ["ensembl_gene_id"],
                                         ["human"]))
            view.map(annotate.annotate_table_with_biomart,
                     *annotate_args)
                     """
            view.map(rseqc.RPKM_saturation, *rseq_args)
            curr_files = tophat_outputs


def find_cores_per_job(config):
    max_threads = config["program"].get("max_threads", 1)
    return max_threads


if __name__ == "__main__":
    # read in the config file and perform initial setup
    main_config_file = sys.argv[1]
    with open(main_config_file) as config_in_handle:
        startup_config = yaml.load(config_in_handle)
    setup_logging(startup_config)
    scheduler = startup_config["cluster"]["scheduler"]
    queue = startup_config["cluster"]["queue"]
    profile = startup_config["cluster"].get("profile", None)
    engines = startup_config["cluster"]["cores"]
    threads = startup_config["program"].get("max_threads", 1)
    cores_per_job = find_cores_per_job(startup_config)

    with cluster.cluster_view(scheduler, queue, engines, cores_per_job, profile) as view:
        main(main_config_file, view)
开发者ID:bachu-mahe,项目名称:projects,代码行数:30,代码来源:rnaseq_pipeline_new.py

示例13: use

                        help="scheduler to use (lsf, sge, torque, slurm, or pbs)")
    parser.add_argument("--queue", dest='queue', required=True,
                        help="queue to use on scheduler.")
    parser.add_argument("--num_jobs", dest='num_jobs', required=True,
                        type=int, help="number of jobs to run in parallel.")
    parser.add_argument("--cores_per_job", dest="cores_per_job", default=1,
                        type=int, help="number of cores for each job.")
    parser.add_argument("--profile", dest="profile", default=None,
                        help="Optional profile to test.")
    parser.add_argument("--resources", dest="resources", default=None,
                        help="Native specification flags to the scheduler")

    args = parser.parse_args()
    args.resources = {'resources': args.resources}

    with cluster_view(args.scheduler, args.queue, args.num_jobs,
                      profile=args.profile, extra_params=args.resources) as view:
        print "First check to see if we can talk to the engines."
        results = view.map(lambda x: "hello world!", range(5))
        print ("This long computation that waits for 5 seconds before returning "
               "takes a while to run serially..")
        start_time = time.time()
        results = map(long_computation, range(20), range(20, 40), range(40, 60))
        print results
        print "That took {0} seconds.".format(time.time() - start_time)

        print "Running it in parallel goes much faster..."
        start_time = time.time()
        results = view.map(long_computation, range(20), range(20, 40), range(40, 60))
        print results
        print "That took {0} seconds.".format(time.time() - start_time)
开发者ID:lbeltrame,项目名称:ipython-cluster-helper,代码行数:31,代码来源:example.py

示例14: load_ipython

def load_ipython(args):
    grabix_file = bgzip(args.vcf)
    with cluster_view(*get_ipython_args(args)) as view:
        chunks = load_chunks_ipython(grabix_file, args, view)
        merge_chunks_ipython(chunks, args, view)
开发者ID:arq5x,项目名称:gemini,代码行数:5,代码来源:gemini_load.py

示例15: main

                    used.append(comp_file)
                    pairs.append([in_file, comp_file])
                    break
        if in_file not in used:
            pairs.append([in_file])
            used.append(in_file)
    return pairs


def main(data_dir, view):
    fastq_files = list(glob.glob(os.path.join(data_dir, "*_trimmed.fixed.fastq")))
    fastq_files = combine_pairs(fastq_files)
    print "Aligning %s." % (fastq_files)
    aligned_files = view.map(align, fastq_files)
    print "Marking duplicates in %s." % (aligned_files)
    marked = view.map(mark_duplicates, aligned_files)
    print "Filtering duplicates and unmapped reads in %s." % (marked)
    deduped = view.map(filter_duplicates, marked)
    #compute_coverage(deduped)
    print "Computing start sites of %s." % (deduped)
    starts = view.map(count_starts, deduped)


if __name__ == "__main__":
    data_dir = sys.argv[1]
    print combine_pairs(list(glob.glob(os.path.join(data_dir, "*_trimmed.fixed.fastq"))))
    fastq_files = combine_pairs(list(glob.glob(os.path.join(data_dir,
                                                            "*_trimmed.fixed.fastq"))))
    with cluster_view("lsf", "hsph_bioinfo", len(fastq_files), cores_per_job=1) as view:
        main(data_dir, view)
开发者ID:Xiuying,项目名称:projects,代码行数:30,代码来源:align_and_call_peaks.py


注:本文中的cluster_helper.cluster.cluster_view函数示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。