Python Job.Job类代码示例

    def testBaggage(self):
        """
        _testBaggage_

        Verify that job baggage is persisted with the package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            baggage = newJob.getBaggage()
            setattr(baggage, "thisJob", newJob["name"])
            setattr(baggage, "seed1", 11111111)
            setattr(baggage, "seed2", 22222222)
            setattr(baggage, "seed3", 33333333)
            setattr(baggage, "seed4", 44444444)
            setattr(baggage, "seed5", 55555555)
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."
            jobBaggage = job.getBaggage()

            assert jobBaggage.thisJob == "Job%d" % i, \
                   "Error: Job baggage has wrong name."
            assert jobBaggage.seed1 == 11111111, \
                   "Error: Job baggee has wrong value for seed1."
            assert jobBaggage.seed2 == 22222222, \
                   "Error: Job baggee has wrong value for seed2."
            assert jobBaggage.seed3 == 33333333, \
                   "Error: Job baggee has wrong value for seed3."
            assert jobBaggage.seed4 == 44444444, \
                   "Error: Job baggee has wrong value for seed4."
            assert jobBaggage.seed5 == 55555555, \
                   "Error: Job baggee has wrong value for seed5."


        return

    def createTestJob(self):
        """
        _createTestJob_

        Create a test job that has parents for each input file.

        """
        newJob = Job(name = "TestJob")
        newJob.addFile(File(lfn = "/some/file/one",
                            parents = set([File(lfn = "/some/parent/one")])))
        newJob.addFile(File(lfn = "/some/file/two",
                            parents = set([File(lfn = "/some/parent/two")])))
        return newJob

    def createTestJob(self):
        """
        Create a test job to pass to the DashboardInterface

        """

        job = Job(name = "ThisIsASillyName")

        testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
        testFileA.addRun(Run(1, *[45]))
        testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
        testFileB.addRun(Run(1, *[46]))

        job.addFile(testFileA)
        job.addFile(testFileB)

        job['id'] = 1

        return job

    def __init__(self, name = None, files = None, id = None):
        """
        ___init___

        jobgroup object is used to determine the workflow.
        inputFiles is a list of files that the job will process.
        """
        WMBSBase.__init__(self)
        WMJob.__init__(self, name = name, files = files)

        self["id"]           = id
        self["jobgroup"]     = None
        self["couch_record"] = None
        self["attachments"]  = {}
        self["cache_dir"]    = None
        self["sandbox"]      = None
        self['fwjr']         = None
        self["mask"]         = Mask()
        self['custom']       = {}  # For local add-ons that we want to send to JSON

        return

    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
                                     CouchDBName = self.testInit.couchDbName,
                                     ConfigCacheDoc = document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
        self.failUnless(seed1 != None)

    def getFiles(self, type="list"):
        """
        _getFiles_

        Retrieve a list of files that are associated with the job.
        """
        if self["id"] == None:
            return WMJob.getFiles(self, type)

        existingTransaction = self.beginTransaction()
        idAction = self.daofactory(classname="Jobs.LoadFiles")
        fileIDs = idAction.execute(self["id"], conn=self.getDBConn(), transaction=self.existingTransaction())

        currentFileIDs = WMJob.getFiles(self, type="id")
        for fileID in fileIDs:
            if fileID["id"] not in currentFileIDs:
                self.loadData()
                break

        self.commitTransaction(existingTransaction)
        return WMJob.getFiles(self, type)

    def associateFiles(self):
        """
        _associateFiles_

        Update the wmbs_job_assoc table with the files in the inputFiles for the
        job.
        """
        files = WMJob.getFiles(self, type="id")

        if len(files) > 0:
            addAction = self.daofactory(classname="Jobs.AddFiles")
            addAction.execute(self["id"], files, conn=self.getDBConn(), transaction=self.existingTransaction())

        return

    def getDataStructsJob(self):
        """
        _getDataStructsJob_

        Returns the DataStructs version of this job
        """
        job = WMJob(name = self['name'])

        # Transfer all simple keys
        for key in self.keys():
            keyType = type(self.get(key))
            if keyType in [str, long, int, float]:
                job[key] = self[key]

        for file in self['input_files']:
            job['input_files'].append(file.returnDataStructsFile())

        job['mask'] = WMMask()
        for key in self["mask"].keys():
            job["mask"][key] = self["mask"][key]

        job.baggage = self.baggage

        return job

    def testC(self):
        """test building a tweak from the seeds"""
        job = Job("TestJob")
        seeder = AutomaticSeeding()

        job.addBaggageParameter("process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
        job.addBaggageParameter("process.RandomNumberGeneratorService.seed44.initialSeed", 98273762)


        seeder(job)

        tweak = PSetTweak()
        for x in job.baggage.process.RandomNumberGeneratorService:
            parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
            tweak.addParameter(parameter, x.initialSeed)
        print(tweak)

    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1,1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
                                                   random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files = self.inputFiles)
        return

    def associateWorkUnits(self):
        """
        _associateWorkUnits_

        Add the WorkUnits that this job requires

        Returns: N/A
        """

        existsAction = self.daofactory(classname='WorkUnit.ExistsByTaskFileLumi')
        addAction = self.daofactory(classname='WorkUnit.Add')
        assocAction = self.daofactory(classname='Jobs.AddWorkUnits')

        files = WMJob.getFiles(self)
        jobMask = self['mask']

        workflow = self.getWorkflow()
        wfid = workflow['taskid']

        lumisInJob = 0
        for wmfile in files:
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                lumisInJob += len(runObj.lumis)

        for wmfile in files:
            fileid = wmfile['id']
            fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
            for runObj in fileMask:
                run = runObj.run
                lumis = runObj.lumis
                for lumi in lumis:
                    if not existsAction.execute(taskid=wfid, fileid=fileid, run_lumi=runObj,
                                                conn=self.getDBConn(), transaction=self.existingTransaction()):
                        addAction.execute(taskid=wfid, last_unit_count=lumisInJob, fileid=fileid, run=run,
                                          lumi=lumi,
                                          conn=self.getDBConn(), transaction=self.existingTransaction())
                    assocAction.execute(jobid=self["id"], fileid=fileid, run=run, lumi=lumi,
                                        conn=self.getDBConn(), transaction=self.existingTransaction())

class JobTest(unittest.TestCase):
    """
    _JobTest_

    Testcase for the Job class

    Instantiate a dummy Job object with a dummy Subscription
    and a dummy Fileset full of random files as input
    """

    def setUp(self):
        """
        _setUp_

        Initial Setup for the Job Testcase
        """
        self.inputFiles = []

        for i in range(1,1000):
            lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
                                                   random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.inputFiles.append(file)

        self.dummyJob = Job(files = self.inputFiles)
        return

    def tearDown(self):
        """
        No tearDown method for this Testcase
        """
        pass

    def testGetFilesList(self):
        """
        _testGetFilesList_

        Verify that the Job::getFiles(type = "list") method returns the same
        files in the same order that they were passed in.
        """
        assert self.dummyJob.getFiles() == self.inputFiles, \
            "ERROR: Initial fileset does not match Job fileset"

        return

    def testGetFilesSet(self):
        """
        _testGetFilesSet_

        Verify that the Job::getFiles(type = "set") method returns the correct
        input files in the form of a set.
        """
        assert self.dummyJob.getFiles(type = "set") == set(self.inputFiles), \
            "ERROR: getFiles(type = 'set') does not work correctly."

        return

    def testGetFilesLFN(self):
        """
        _testGetFilesLFN_

        Verify that the Job::getFiles(type = "lfn") method returns the same
        files in the same order that they were passed in.
        """
        jobLFNs = self.dummyJob.getFiles(type = "lfn")

        goldenLFNs = []
        for file in self.inputFiles:
            goldenLFNs.append(file["lfn"])

        assert len(goldenLFNs) == len(jobLFNs), \
            "ERROR: Job has different number of files than input"

        for jobLFN in jobLFNs:
            assert jobLFN in goldenLFNs, \
                "ERROR: LFN missing from job."

        return

    def testAddFile(self):
        """
        _testAddFile_

        Verify that the Job::addFile() method works properly.
        """
        dummyFileAddFile = File("/tmp/dummyFileAddFileTest", 1234, 1, 2)
        self.dummyJob.addFile(dummyFileAddFile)

        assert dummyFileAddFile in self.dummyJob.getFiles(), \
            "ERROR: Couldn't add file to Job - addFile method error"

        return

    def testChangeState(self):
#.........这里部分代码省略.........