Python Fileset.getFiles方法代码示例

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
 def testCommit(self):
     """
         Testcase for the commit method of the Fileset class
         
     """
     localTestFileSet = Fileset('LocalTestFileset', self.initialSet)
     fsSize = len(localTestFileSet.getFiles(type = "lfn"))
     #Dummy file to test
     fileTestCommit = File('/tmp/filetestcommit',0000,1,1)
     #File is added to the newfiles attribute of localTestFileSet
     localTestFileSet.addFile(fileTestCommit)
     assert fsSize == len(localTestFileSet.getFiles(type = "lfn")) - 1, 'file not added'\
             'correctly to test fileset'
     newfilestemp = localTestFileSet.newfiles
     assert fileTestCommit in newfilestemp, 'test file not in the new files'\
             'list' 
     #After commit, dummy file is supposed to move from newfiles to files
     localTestFileSet.commit()
     #First, testing if the new file is present at file set object attribute of the Fileset object
     
     assert newfilestemp.issubset(localTestFileSet.files), 'Test file not ' \
             'present at fileset.files - fileset.commit ' \
             'not working properly' 
     #Second, testing if the newfile set object attribute is empty
     assert localTestFileSet.newfiles == set(), \
             'Test file not present at fileset.newfiles ' \
             '- fileset.commit not working properly'

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
 def testCall(self):
     fileset = Fileset(name="FakeFeederTest")
     for i in range(1, 21):
         self.feeder([fileset])
         set = fileset.getFiles(type = "set")
         if len(set) > 0:
             file = set.pop()
         fileset.commit()

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
class Subscription(Pickleable, dict):
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def workflowType(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].wfType    

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def owner(self):
        if self['workflow'] == None:
            return 'Unknown'
        return self['workflow'].owner

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files = [], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
class JobGroup(WMObject):
    """
    JobGroups are sets of jobs running on files who's output needs to be merged
    together.
    """
    def __init__(self, subscription = None, jobs = None):
        self.jobs = []
        self.newjobs = []
        self.id = 0
        
        if type(jobs) == list:
            self.newjobs = jobs
        elif jobs != None:
            self.newjobs = [jobs]
            
        self.subscription = subscription
        self.output = Fileset()
        self.last_update = datetime.datetime.now()
        
    def add(self, job):
        """
        _add_

        Add a Job or list of jobs to the JobGroup.
        """
        jobList = self.makelist(job)
        self.newjobs.extend(jobList)
        return

    def commit(self):
        """
        _commit_

        Move any jobs in the newjobs dict to the job dict.  Empty the newjobs
        dict.
        """
        self.jobs.extend(self.newjobs)
        self.newjobs = []
    
    def commitBulk(self):
        """
        Dummy method for consistency with WMBS implementation
        """
        self.commit()
        
    def addOutput(self, file):
        """
        _addOutput_

        Add a File to the JobGroup's output fileset.  The File is committed
        to the Fileset immediately.
        """
        self.output.addFile(file)
        self.output.commit()

    def getJobs(self, type = "list"):
        """
        _getJobs_

        Retrieve all of the jobs in the JobGroup.  The output will either be
        returned as a list of Job objects (when type is "list") or a list of
        Job IDs (when type is "id").
        """
        if type == "list":
            return self.jobs
        elif type == "id":
            jobIDs = []

            for job in self.jobs:
                jobIDs.append(job["id"])

            return jobIDs
        else:
            print "Unknown type: %s" % type

        return

    def getOutput(self, type = "list"):
        """
        _getOutput_

        Retrieve all of the files that are in the JobGroup's output fileset.
        Type can be one of the following: list, set, lfn, id.
        """
        return self.output.getFiles(type = type)

    def getLength(self, obj):
        """
        This just gets a length for either dict or list objects
        """
        if type(obj) == dict:
            return len(obj.keys())
        elif type(obj) == list:
            return len(obj)
        else:
            return 0

    def __len__(self):
        """
        Allows use of len() on JobGroup
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
class SubscriptionTest(unittest.TestCase):
    """
    _SubscriptionTest_

    Testcase for the Subscription class

    """

    def setUp(self):
        """
        Initial Setup for Subscription Testcase

        Set a dummy Subscription with a fileset composed of one file inside it
        and a dummy workflow using the default constructor of the Workflow class

        """
        self.dummyFile = File('/tmp/dummyfile',9999,0,0,0)
        self.dummySet = set()
        self.dummySet.add(self.dummyFile)
        self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset',
                                    files = self.dummySet)
        self.dummyWorkFlow = Workflow()
        self.dummySubscription = Subscription(fileset = self.dummyFileSet,
                                              workflow = self.dummyWorkFlow)
        return

    def tearDown(self):
        pass

    def testGetWorkflow(self):
        """
        Testcase for the getWorkflow method of the Subscription Class

        """
        assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
        'Couldn\'t add Workflow to Subscription'

    def testGetFileset(self):
        """
        Testcase for the getFileset method of the Subscription Class

        """
        assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
        'Couldn\'t add Fileset to Subscription - name does not match'

        for x in self.dummyFileSet.listNewFiles():
            assert x in self.dummySubscription['fileset'].newfiles, \
            'Couldn\'t add Fileset to Subscription - newFiles Set does not match'

        assert self.dummyFileSet.getFiles(type='set') == \
            self.dummySubscription['fileset'].getFiles(type='set'), \
            'Couldn\'t add Fileset to Subscription - %s Set does not match' % x

    def testAcquireFiles(self):
        """
        Testcase for the acquireFiles method of the Subscription Class

        """
        #Cleaning possible files already occupying the available set
        self.dummySubscription.acquireFiles()

        # First test - Test if initial file (on available set) is inserted in the
        # acquired set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
        #Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.available.listNewFiles()
        #Check if Set returned by method is the same that was at the previous
        #available FileSet
        assert S == self.dummySubscription.acquireFiles(), \
        'Couldn\'t acquire file using method acquireFiles - (no arguments test)'


        #Second test - Test if target files are inserted at the acquired set

        dummyFileList = set()
        #Populating the dummy List with a random number of files
        for i in range(1, random.randint(100,1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events,
                        checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.add(file)

        #Check if return value is correct - with parameters
        acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList)
        assert acqFiles == dummyFileList,\
                'Return value for acquireFiles method not the acquired files'
        #Check if all files were inserted at subscription acquired files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
            'Couldn\'t acquire File %s' % x.dict['lfn']
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import getFiles [as 别名]
class FilesetTest (unittest.TestCase):
    """ 
    _FilesetTest_

    Testcase for Fileset
    """
    def setUp(self):
        """
        Create a dummy fileset and populate it with random files,
        in order to use it for the testcase methods
        
        """
        logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=__file__.replace('.py','.log'),
                    filemode='w')
        self.logger = logging.getLogger('FilesetClassTest')
        
        #Setup the initial testcase environment:
        initialfile = File('/tmp/lfn1',1000,1,1,1)
        self.initialSet = set()
        self.initialSet.add(initialfile)
        
        #Create a Fileset, containing a initial file on it.
        self.fileset = Fileset(name = 'self.fileset', files = self.initialSet)
        #Populate the fileset with random files
        for i in range(1,1000):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn, size=size, events=events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.fileset.addFile(file)
        
    def tearDown(self):
        """
            No tearDown method for this testcase
            
        """
        pass
    def testAddFile(self):
        """
            Testcase for the addFile method of the Fileset class
            
        """
        #First test - Add file and check if its there
        testfile = File('/tmp/lfntest',9999,9,9)
        self.fileset.addFile(testfile)
        assert(testfile in self.fileset.listNewFiles(), 'Couldn\'t add file ' +
                'to fileset - fileset.addfile method not working')
        #Second test - Add file that was already at Fileset.files , 
        # and check if it gets updated
        testFileSame = File('/tmp/lfntest',9999,9,9)
        testFileSame.setLocation(set('dummyse.dummy.com'))
        self.fileset.addFile(testFileSame)
        assert(testFileSame in  self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile not updating location of already ' +
               'existing files' )
        assert(testfile in self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile unable to remove previous file ' +
               'from list')
        #Third test - Add file that was already at Fileset.newfiles , 
        #and check if it gets updated
        assert(testFileSame in  self.fileset.listNewFiles(),'Same file copy ' +
               'failed - fileset.addFile not adding file to fileset.newFiles')
        
    def testListFiles(self):
        """
        _testListFiles_
        
        Testcase for the getFiles() method of the Fileset class
        """
        filesettemp = self.fileset.getFiles()
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetFiles(self):
        """
        Check that all files returned by the set are the same as those added to 
        the fileset
        """
        filesettemp = self.fileset.getFiles(type = "set")
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetListCompare(self):
        """
        Test that all files in fileset.setFiles are in fileset.getFiles()
        """
        thelist = self.fileset.getFiles()
        theset = self.fileset.getFiles(type = "set")
        for x in thelist:
            assert x in (theset), \
#.........这里部分代码省略.........