Python models.RunnableTask類代碼示例

def write_task_manifest(manifest_path, tid, task, resource_types, task_version, python_mode_str, cluster_renderer):
    """

    :type manifest_path: str
    :type tid: str
    :type task: Task
    :type task_version: str
    :type python_mode_str: str
    :type cluster_renderer: ClusterTemplateRender | None

    :return:
    """
    # this should be already set in task, or is this for deferred reasons?
    resources_list_d = [dict(resource_type=r, path=p) for r, p in zip(resource_types, task.resources)]

    task_manifest = os.path.join(manifest_path)

    with open(task_manifest, 'w+') as f:
        # this version should be the global pbsmrtpipe version
        # or the manifest file spec version?
        runnable_task = RunnableTask(task, cluster_renderer)
        f.write(json.dumps(runnable_task.to_dict(), sort_keys=True, indent=2))

    log.debug("wrote task id {i} to {p}".format(i=tid, p=manifest_path))
    return True

def run_task_manifest_on_cluster(path):
    """
    Run the Task on the queue (of possible)

    :param path:
    :return:
    """
    output_dir = os.path.dirname(path)
    os.chdir(output_dir)
    rt = RunnableTask.from_manifest_json(path)

    # this needs to be updated to have explicit paths to stderr, stdout
    rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)
    cstderr = os.path.join(output_dir, "cluster.stderr")
    stderr = os.path.join(output_dir, "stderr")

    state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
    # Need to update the run_task_on_cluster
    emsg = ""
    if rcode != 0:
        # try to provide a hint of the exception from the stderr
        detail_msg = _extract_last_nlines(stderr)
        cdetails_msg = _extract_last_nlines(cstderr)
        emsg = "{i} Failed with exit code {r} {c}\n{x}".format(r=rcode, i=rt.task.task_id, x=detail_msg, c=cdetails_msg)

    return state, err_msg, run_time

def _args_run_task_manifest(args):
    output_dir = os.getcwd() if args.output_dir is None else args.output_dir
    task_manifest_path = args.task_manifest

    rt = RunnableTask.from_manifest_json(task_manifest_path)

    rcode, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr, args.debug)

    return rcode

def _args_run_task_manifest(args):
    output_dir = os.getcwd() if args.output_dir is None else args.output_dir
    task_manifest_path = args.task_manifest

    log.info("Loading runnable-task from {f}".format(f=task_manifest_path))
    rt = RunnableTask.from_manifest_json(task_manifest_path)
    log.info("loaded runnable-task")

    # (exit code, run_time_sec) =
    rcode, err_msg, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr, args.debug)

    return rcode

def run_task_manifest_on_cluster(path):
    """
    Run the Task on the queue (of possible)

    :param path:
    :return:
    """
    output_dir = os.path.dirname(path)
    rt = RunnableTask.from_manifest_json(path)

    rcode, run_time = run_task_on_cluster(rt, path, output_dir, True)

    state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
    msg = "{r} failed".format(r=rt) if rcode != 0 else ""

    return state, msg, run_time

def run_task_manifest_on_cluster(path):
    """
    Run the Task on the queue (of possible)

    :param path:
    :return:
    """
    output_dir = os.path.dirname(path)
    os.chdir(output_dir)
    rt = RunnableTask.from_manifest_json(path)

    # this needs to be updated to have explicit paths to stderr, stdout
    rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)

    state = TaskStates.from_int(rcode)

    return state, err_msg, run_time

def run_task_manifest(path):
    output_dir = os.path.dirname(path)
    stderr = os.path.join(output_dir, 'stderr')
    stdout = os.path.join(output_dir, 'stdout')

    try:
        rt = RunnableTask.from_manifest_json(path)
    except KeyError:
        emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
        log.error(emsg)
        raise

    rcode, run_time = run_task(rt, output_dir, stdout, stderr, True)

    state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
    msg = "" if rcode == 0 else "Failed with exit code {r}".format(r=rcode)

    return state, msg, run_time

def run_task_manifest(path):
    output_dir = os.path.dirname(path)
    os.chdir(output_dir)
    stderr = os.path.join(output_dir, 'stderr')
    stdout = os.path.join(output_dir, 'stdout')

    try:
        rt = RunnableTask.from_manifest_json(path)
    except KeyError:
        emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
        log.error(emsg)
        raise

    # blocking call
    rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)

    state = TaskStates.from_int(rcode)

    return state, err_msg, run_time

def run_task_manifest(path):
    output_dir = os.path.dirname(path)
    os.chdir(output_dir)
    stderr = os.path.join(output_dir, 'stderr')
    stdout = os.path.join(output_dir, 'stdout')

    try:
        rt = RunnableTask.from_manifest_json(path)
    except KeyError:
        emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
        log.error(emsg)
        raise

    rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)

    state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
    emsg = ""
    if rcode != 0:
        # try to provide a hint of the exception from the stderr
        detail_msg = _extract_last_nlines(stderr)
        emsg = "{i} Failed with exit code {r} {x}".format(r=rcode, i=rt.task.task_id, x=detail_msg)

    return state, err_msg, run_time

#.........這裏部分代碼省略.........
                bg.node[tnode]['task'] = task
                tnode_to_task[tnode] = task

                if isinstance(tnode.meta_task, (ToolContractMetaTask, ScatterToolContractMetaTask, GatherToolContractMetaTask)):
                    # the task.options have actually already been resolved here, but using this other
                    # code path for clarity
                    if isinstance(tnode.meta_task, ToolContractMetaTask):
                        rtc = IO.static_meta_task_to_rtc(tnode.meta_task, task, task_opts, task_dir, tmp_dir, max_nproc)
                    elif isinstance(tnode.meta_task, ScatterToolContractMetaTask):
                        rtc = IO.static_scatter_meta_task_to_rtc(tnode.meta_task, task, task_opts, task_dir, tmp_dir, max_nproc, max_nchunks, tnode.meta_task.chunk_keys)
                    elif isinstance(tnode.meta_task, GatherToolContractMetaTask):
                        # this should always be a TaskGatherBindingNode which will have a .chunk_key
                        rtc = IO.static_gather_meta_task_to_rtc(tnode.meta_task, task, task_opts, task_dir, tmp_dir, max_nproc, tnode.chunk_key)
                    else:
                        raise TypeError("Unsupported task type {t}".format(t=tnode.meta_task))

                    # write driver manifest, which calls the resolved-tool-contract.json
                    # there's too many layers of indirection here. Partly due to the pre-tool-contract era
                    # python defined tasks.
                    # Always write the RTC json for debugging purposes
                    tc_path = os.path.join(task_dir, GlobalConstants.TOOL_CONTRACT_JSON)
                    write_tool_contract(tnode.meta_task.tool_contract, tc_path)

                    rtc_json_path = os.path.join(task_dir, GlobalConstants.RESOLVED_TOOL_CONTRACT_JSON)
                    rtc_avro_path = os.path.join(task_dir, GlobalConstants.RESOLVED_TOOL_CONTRACT_AVRO)
                    if rtc.driver.serialization == 'avro':
                        # hack to fix command
                        task.cmds[0] = task.cmds[0].replace('.json', '.avro')
                        write_resolved_tool_contract_avro(rtc, rtc_avro_path)
                    # for debugging
                    write_resolved_tool_contract(rtc, rtc_json_path)

                runnable_task_path = os.path.join(task_dir, GlobalConstants.RUNNABLE_TASK_JSON)
                runnable_task = RunnableTask(task, global_registry.cluster_renderer)
                runnable_task.write_json(runnable_task_path)

                # Create an instance of Worker
                w = _to_worker(tnode.meta_task.is_distributed, "worker-task-{i}".format(i=tid), tid, runnable_task_path)

                workers[tid] = w
                w.start()
                total_nproc += task.nproc
                slog.info("Starting worker {i} ({n} workers running, {m} total proc in use)".format(i=tid, n=len(workers), m=total_nproc))

                # Submit job to be run.
                B.update_task_state(bg, tnode, TaskStates.SUBMITTED)
                msg_ = "Updating task {t} to SUBMITTED".format(t=tid)
                log.debug(msg_)
                tid_to_tnode[tid] = tnode
                services_log_update_progress("pbsmrtpipe::{i}".format(i=tnode.idx), WS.LogLevels.INFO, msg_)
                BU.write_binding_graph_images(bg, job_resources.workflow)

            elif isinstance(tnode, EntryOutBindingFileNode):
                # Handle EntryPoint types. This is not a particularly elegant design :(
                bg.node[tnode]['nproc'] = 1
                log.info("Marking task as completed {t}".format(t=tnode))
                B.update_task_state_to_success(bg, tnode, 0.0)
                # Update output paths
                mock_file_index = 0
                for fnode in bg.successors(tnode):
                    file_path = "/path/to/mock-file-{i}.txt".format(i=mock_file_index)
                    B.update_file_state_to_resolved(bg, fnode, file_path)
                    mock_file_index += 1
            else:
                raise TypeError("Unsupported node type {t} of '{x}'".format(t=type(tnode), x=tnode))

def _args_pprint_task_manifest(args):
    return pprint_task_manifest(RunnableTask.from_manifest_json(args.task_manifest))

def _args_to_cmd(args):
    return run_to_cmd(RunnableTask.from_manifest_json(args.task_manifest))

def validate_file_and_load_manifest(path):
    rt = RunnableTask.from_manifest_json(validate_file(path))
    # if we got here everything is valid
    return path

 def test_task_manifest_serialization(self):
     path = os.path.join(TEST_DATA_DIR, 'task-manifest.json')
     r = RunnableTask.from_manifest_json(path)
     self.assertIsInstance(r, RunnableTask)