Python Fileset.commit方法代码示例

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
 def testCommit(self):
     """
         Testcase for the commit method of the Fileset class
         
     """
     localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
     fsSize = len(localTestFileSet.getFiles(type = "lfn"))
     #Dummy file to test
     fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
     #File is added to the newfiles attribute of localTestFileSet
     localTestFileSet.addFile(fileTestCommit)
     assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
             'correctly to test fileset'
     newfilestemp = localTestFileSet.newfiles
     assert fileTestCommit in newfilestemp, 'test file not in the new files'\
             'list' 
     #After commit, dummy file is supposed to move from newfiles to files
     localTestFileSet.commit()
     #First, testing if the new file is present at file set object attribute of the Fileset object
     
     assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
             'present at fileset.files - fileset.commit ' \
             'not working properly' 
     #Second, testing if the newfile set object attribute is empty
     assert localTestFileSet.newfiles == set(), \
             'Test file not present at fileset.newfiles ' \
             '- fileset.commit not working properly'

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
    def testMetaData(self):
        """
        _testMetaData_

        Make sure that the workflow name, task, owner and white and black lists
        make it into each job object.
        """
        testWorkflow = Workflow(spec = "spec.pkl", owner = "Steve",
                                name = "TestWorkflow", task = "TestTask")

        testFileset = Fileset(name = "TestFileset")
        testFile = File(lfn = "someLFN")
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset = testFileset,
                                        workflow = testWorkflow,
                                        split_algo = "FileBased")

        myJobFactory = JobFactory(subscription = testSubscription)
        testJobGroups =  myJobFactory(siteWhitelist = ["site1"], siteBlacklist = ["site2"])
        self.assertTrue(len(testJobGroups) > 0)

        for testJobGroup in testJobGroups:
            self.assertTrue(len(testJobGroup.jobs) > 0)
            for job in testJobGroup.jobs:
                self.assertEqual(job["task"], "TestTask", "Error: Task is wrong.")
                self.assertEqual(job["workflow"], "TestWorkflow", "Error: Workflow is wrong.")
                self.assertEqual(job["owner"], "Steve", "Error: Owner is wrong.")
        return

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
 def testCall(self):
     fileset = Fileset(name="FakeFeederTest")
     for i in range(1, 21):
         self.feeder([fileset])
         set = fileset.getFiles(type = "set")
         if len(set) > 0:
             file = set.pop()
         fileset.commit()

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
    def testProductionRunNumber(self):
        """
        _testProductionRunNumber_
        
        Verify that jobs created by production subscritpions have the correct
        run number is their job mask.  Also verify that non-production
        subscriptions don't have modified run numbers.
        """
        testWorkflow = Workflow(spec = "spec.pkl", owner = "Steve",
                                name = "TestWorkflow", task = "TestTask")
    
        testFileset = Fileset(name = "TestFileset")
        testFile = File(lfn = "someLFN")
        testFileset.addFile(testFile)
        testFileset.commit()
        
        testSubscription = Subscription(fileset = testFileset,
                                        workflow = testWorkflow,
                                        split_algo = "FileBased",
                                        type = "Production")
    
        myJobFactory = JobFactory(subscription = testSubscription)
        testJobGroups =  myJobFactory()
    
        for testJobGroup in testJobGroups:
            for job in testJobGroup.jobs:
                assert job["mask"]["FirstRun"] == 1, \
                       "Error: First run is wrong."
                assert job["mask"]["LastRun"] == 1, \
                       "Error: Last run is wrong."

        testSubscription = Subscription(fileset = testFileset,
                                        workflow = testWorkflow,
                                        split_algo = "FileBased",
                                        type = "Processing")
    
        myJobFactory = JobFactory(subscription = testSubscription)
        testJobGroups =  myJobFactory()
        
        for testJobGroup in testJobGroups:
            for job in testJobGroup.jobs:
                assert job["mask"]["FirstRun"] == None, \
                       "Error: First run is wrong."
                assert job["mask"]["LastRun"] == None, \
                       "Error: Last run is wrong."

        return

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
class Subscription(Pickleable, dict):
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def workflowType(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].wfType    

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def owner(self):
        if self['workflow'] == None:
            return 'Unknown'
        return self['workflow'].owner

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files = [], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import commit [as 别名]
class JobGroup(WMObject):
    """
    JobGroups are sets of jobs running on files who's output needs to be merged
    together.
    """
    def __init__(self, subscription = None, jobs = None):
        self.jobs = []
        self.newjobs = []
        self.id = 0
        
        if type(jobs) == list:
            self.newjobs = jobs
        elif jobs != None:
            self.newjobs = [jobs]
            
        self.subscription = subscription
        self.output = Fileset()
        self.last_update = datetime.datetime.now()
        
    def add(self, job):
        """
        _add_

        Add a Job or list of jobs to the JobGroup.
        """
        jobList = self.makelist(job)
        self.newjobs.extend(jobList)
        return

    def commit(self):
        """
        _commit_

        Move any jobs in the newjobs dict to the job dict.  Empty the newjobs
        dict.
        """
        self.jobs.extend(self.newjobs)
        self.newjobs = []
    
    def commitBulk(self):
        """
        Dummy method for consistency with WMBS implementation
        """
        self.commit()
        
    def addOutput(self, file):
        """
        _addOutput_

        Add a File to the JobGroup's output fileset.  The File is committed
        to the Fileset immediately.
        """
        self.output.addFile(file)
        self.output.commit()

    def getJobs(self, type = "list"):
        """
        _getJobs_

        Retrieve all of the jobs in the JobGroup.  The output will either be
        returned as a list of Job objects (when type is "list") or a list of
        Job IDs (when type is "id").
        """
        if type == "list":
            return self.jobs
        elif type == "id":
            jobIDs = []

            for job in self.jobs:
                jobIDs.append(job["id"])

            return jobIDs
        else:
            print "Unknown type: %s" % type

        return

    def getOutput(self, type = "list"):
        """
        _getOutput_

        Retrieve all of the files that are in the JobGroup's output fileset.
        Type can be one of the following: list, set, lfn, id.
        """
        return self.output.getFiles(type = type)

    def getLength(self, obj):
        """
        This just gets a length for either dict or list objects
        """
        if type(obj) == dict:
            return len(obj.keys())
        elif type(obj) == list:
            return len(obj)
        else:
            return 0

    def __len__(self):
        """
        Allows use of len() on JobGroup
#.........这里部分代码省略.........