Python Job.wrapJobFn方法代码示例

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
    def runNewCheckpointIsLeafVertexTest(self, createWorkflowFn):
        """
        Test verification that a checkpoint job is a leaf vertex using both
        valid and invalid cases.

        :param createWorkflowFn: function to create and new workflow and return a tuple of:

                                 0) the workflow root job
                                 1) a checkpoint job to test within the workflow

        """

        logger.info('Test checkpoint job that is a leaf vertex')
        self.runCheckpointVertexTest(*createWorkflowFn(),
                                     expectedException=None)

        logger.info('Test checkpoint job that is not a leaf vertex due to the presence of a service')
        self.runCheckpointVertexTest(*createWorkflowFn(),
                                     checkpointJobService=TrivialService("LeafTestService"),
                                     expectedException=JobGraphDeadlockException)

        logger.info('Test checkpoint job that is not a leaf vertex due to the presence of a child job')
        self.runCheckpointVertexTest(*createWorkflowFn(),
                                     checkpointJobChild=Job.wrapJobFn(
                                         simpleJobFn, "LeafTestChild"),
                                     expectedException=JobGraphDeadlockException)

        logger.info('Test checkpoint job that is not a leaf vertex due to the presence of a follow-on job')
        self.runCheckpointVertexTest(*createWorkflowFn(),
                                     checkpointJobFollowOn=Job.wrapJobFn(
                                         simpleJobFn,
                                         "LeafTestFollowOn"),
                                     expectedException=JobGraphDeadlockException)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def _deleteLocallyReadFilesFn(self, readAsMutable):
     self.options.retryCount = 0
     A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, memory='10M')
     B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable,
                       memory='20M')
     A.addChild(B)
     Job.Runner.startToil(A, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testCacheEjection(self):
     """
     Test cache always always ejects least recently created file
     """
     # Makes three jobs that create an output file each which they write to filestore.  The combined size of any two
     # files is always less that cacheSize but the combined size of all 3 is always more so 1 file always has to be
     # ejected. Test to ensure that A is always ejected regardless of size.
     #  Make a temp directory for the test
     test_dir = self._createTempDir()
     for test in xrange(10):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.logLevel = "DEBUG"
         options.cacheSize = 100000
         options.retryCount=100
         options.badWorker=0.5
         options.badWorkerFailInterval = 1.0
         # Create a temp file to write teh test results
         handle, logfile = tempfile.mkstemp(dir=test_dir)
         os.close(handle)
         file_sizes = [50000, 40000, 30000]
         # Randomize to (potentially) test all combinations
         random.shuffle(file_sizes)
         # Run the workflow. A, B and C do teh cache operations, and D prints test status to tempFile
         A = Job.wrapJobFn(fileTestJob, file_sizes[0])
         B = Job.wrapJobFn(fileTestJob, file_sizes[0])
         C = Job.wrapJobFn(fileTestJob, file_sizes[0])
         D = Job.wrapJobFn(fileTestCache, A.rv(), B.rv(), C.rv(), logfile)
         A.addChild(B)
         B.addChild(C)
         C.addChild(D)
         Job.Runner.startToil(A, options)
         #  Assert jobs passed by reading test results from tempFile
         with open(logfile, 'r') as outfile:
             for test_status in outfile:
                 assert test_status.strip() == 'True'

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testPromiseRequirementRaceStatic(self):
     """
     Checks for a race condition when using promised requirements and child job functions.
     """
     A = Job.wrapJobFn(logDiskUsage, 'A', sleep=5, disk=PromisedRequirement(1024))
     B = Job.wrapJobFn(logDiskUsage, 'B', disk=PromisedRequirement(lambda x: x + 1024, A.rv()))
     A.addChild(B)
     Job.Runner.startToil(A, self.getOptions(self._createTempDir('testFiles')))

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testReadCachHitFileFromJobStore(self):
     """
     Read a file from the file store that has a corresponding cached copy.  Ensure the number
     of links on the file are appropriate.
     """
     A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True)
     B = Job.wrapJobFn(self._readFromJobStore, isCachedFile=True, cacheReadFile=None,
                       fsID=A.rv())
     A.addChild(B)
     Job.Runner.startToil(A, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
        def _testCacheMissFunction(self, cacheReadFile):
            """
            This is the function that actually does what the 2 cache miss functions want.

            :param cacheReadFile: Does the read file need to be cached(T) or not(F)
            """
            workdir = self._createTempDir(purpose='nonLocalDir')
            A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=False, nonLocalDir=workdir)
            B = Job.wrapJobFn(self._readFromJobStore, isCachedFile=False,
                              cacheReadFile=cacheReadFile, fsID=A.rv())
            A.addChild(B)
            Job.Runner.startToil(A, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def _deleteLocallyReadFilesFn(self, readAsMutable):
     self.options.retryCount = 0
     A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True, memory='10M')
     B = Job.wrapJobFn(self._removeReadFileFn, A.rv(), readAsMutable=readAsMutable, memory='20M')
     A.addChild(B)
     try:
         Job.Runner.startToil(A, self.options)
     except FailedJobsException as err:
         self.assertEqual(err.numberOfFailedJobs, 2)
         errMsg = self._parseAssertionError(self.options.logFile)
         if 'explicitly' not in errMsg:
             self.fail('Shouldn\'t see this')

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testControlledFailedWorkerRetry(self):
     """
     Conduct a couple of job store operations.  Then die.  Ensure that the restarted job is
     tracking values in the cache state file appropriately.
     """
     workdir = self._createTempDir(purpose='nonLocalDir')
     self.options.retryCount = 1
     F = Job.wrapJobFn(self._controlledFailTestFn, jobDisk=2*1024*1024*1024, testDir=workdir,
                       disk='2G')
     G = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk='100M')
     F.addChild(G)
     Job.Runner.startToil(F, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testToilIsNotBroken(self):
     """
     Runs a simple DAG to test if if any features other that caching were broken.
     """
     A = Job.wrapJobFn(self._uselessFunc)
     B = Job.wrapJobFn(self._uselessFunc)
     C = Job.wrapJobFn(self._uselessFunc)
     D = Job.wrapJobFn(self._uselessFunc)
     A.addChild(B)
     A.addChild(C)
     B.addChild(D)
     C.addChild(D)
     Job.Runner.startToil(A, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def test_star(self):
     """
     Test the functionality of align_dna
     """
     univ_options = self._getTestUnivOptions()
     config_file = os.path.join(self._projectRootPath(), "src/protect/test/test_inputs/ci_parameters.yaml")
     test_src_folder = os.path.join(self._projectRootPath(), "src", "protect", "test")
     a = Job.wrapJobFn(self._get_test_star_files)
     b = Job.wrapJobFn(self._get_all_tools, config_file).encapsulate()
     c = Job.wrapJobFn(self._get_tool, b.rv(), "star")
     d = Job.wrapJobFn(align_rna, a.rv(), univ_options, c.rv()).encapsulate()
     a.addChild(b)
     b.addChild(c)
     c.addChild(d)
     Job.Runner.startToil(a, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
            def userScript():
                from toil.job import Job
                from toil.common import Toil

                # A user-defined type, i.e. a type defined in the user script
                class X(object):
                    pass

                # noinspection PyUnusedLocal
                def job(job, x, disk='10M', cores=1, memory='10M'):
                    return x

                if __name__ == '__main__':
                    options = Job.Runner.getDefaultArgumentParser().parse_args()
                    x = X()
                    with Toil(options) as toil:
                        r = toil.start(Job.wrapJobFn(job, x).encapsulate())
                    # Assert that the return value is of type X, but not X from the __main__
                    # module but X from foo.bar, the canonical name for the user module. The
                    # translation from __main__ to foo.bar is a side effect of hot-deployment.
                    assert r.__class__ is not X
                    import foo.bar
                    assert r.__class__ is foo.bar.X
                    # Assert that a copy was made. This is a side effect of pickling/unpickling.
                    assert x is not r

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def test_mhc_assessment(self):
     """
     Test the functionality of assess_mhc_genes
     """
     univ_options = self._getTestUnivOptions()
     test_src_folder = os.path.join(self._projectRootPath(), 'src', 'protect', 'test')
     a = Job.wrapJobFn(self._get_test_rsem_file, test_src_folder)
     b = Job.wrapJobFn(self._get_MHC_file)
     c = Job.wrapJobFn(self._get_test_haplotype_file, test_src_folder)
     d = Job.wrapJobFn(assess_mhc_genes, a.rv(), c.rv(), univ_options, b.rv())
     e = Job.wrapJobFn(self._test_output, d.rv(), univ_options)
     a.addChild(b)
     b.addChild(c)
     c.addChild(d)
     d.addChild(e)
     Job.Runner.startToil(a, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
def main(args):
    
    options = parse_args(args) 
    
    RealTimeLogger.start_master()

    for graph in options.graphs:
        if os.path.splitext(graph)[1] != ".vg":
            raise RuntimeError("Input graphs expected to have .vg extension")

    # Make a root job
    root_job = Job.wrapJobFn(compute_kmer_indexes, options,
        cores=1, memory="2G", disk=0)
    
    # Run it and see how many jobs fail
    if not options.only_summary:
        failed_jobs = Job.Runner.startToil(root_job,  options)
    else:
        failed_jobs = 0
    
    if failed_jobs > 0:
        raise Exception("{} jobs failed!".format(failed_jobs))
                               
    RealTimeLogger.stop_master()

    # Do the drawing outside toil to get around weird import problems
    cluster_comparisons(options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
 def testWriteLocalFileToJobStore(self):
     """
     Write a file from the localTempDir to the job store.  Such a file will be cached by
     default.  Ensure the file is cached.
     """
     A = Job.wrapJobFn(self._writeFileToJobStore, isLocalFile=True)
     Job.Runner.startToil(A, self.options)

# 需要导入模块: from toil.job import Job [as 别名]
# 或者: from toil.job.Job import wrapJobFn [as 别名]
def main(args):
    """
    Parses command line arguments and do the work of the program.
    "args" specifies the program arguments, with args[0] being the executable
    name. The return value should be used as the program's exit code.
    """
    
    if len(args) == 2 and args[1] == "--test":
        # Run the tests
        return doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
    
    options = parse_args(args) # This holds the nicely-parsed options object
    
    RealTimeLogger.start_master()
    
    # Make a root job
    root_job = Job.wrapJobFn(collate_all, options,
        cores=1, memory="1G", disk="1G")
    
    # Run it and see how many jobs fail
    failed_jobs = Job.Runner.startToil(root_job,  options)
    
    if failed_jobs > 0:
        raise Exception("{} jobs failed!".format(failed_jobs))
        
    print("All jobs completed successfully")
    
    RealTimeLogger.stop_master()