Python Fileset.listNewFiles方法代码示例

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import listNewFiles [as 别名]
class SubscriptionTest(unittest.TestCase):
    """
    _SubscriptionTest_

    Testcase for the Subscription class

    """

    def setUp(self):
        """
        Initial Setup for Subscription Testcase

        Set a dummy Subscription with a fileset composed of one file inside it
        and a dummy workflow using the default constructor of the Workflow class

        """
        self.dummyFile = File('/tmp/dummyfile',9999,0,0,0)
        self.dummySet = set()
        self.dummySet.add(self.dummyFile)
        self.dummyFileSet = Fileset(name = 'SubscriptionTestFileset',
                                    files = self.dummySet)
        self.dummyWorkFlow = Workflow()
        self.dummySubscription = Subscription(fileset = self.dummyFileSet,
                                              workflow = self.dummyWorkFlow)
        return

    def tearDown(self):
        pass

    def testGetWorkflow(self):
        """
        Testcase for the getWorkflow method of the Subscription Class

        """
        assert self.dummySubscription['workflow'] == self.dummyWorkFlow, \
        'Couldn\'t add Workflow to Subscription'

    def testGetFileset(self):
        """
        Testcase for the getFileset method of the Subscription Class

        """
        assert self.dummyFileSet.name == self.dummySubscription['fileset'].name, \
        'Couldn\'t add Fileset to Subscription - name does not match'

        for x in self.dummyFileSet.listNewFiles():
            assert x in self.dummySubscription['fileset'].newfiles, \
            'Couldn\'t add Fileset to Subscription - newFiles Set does not match'

        assert self.dummyFileSet.getFiles(type='set') == \
            self.dummySubscription['fileset'].getFiles(type='set'), \
            'Couldn\'t add Fileset to Subscription - %s Set does not match' % x

    def testAcquireFiles(self):
        """
        Testcase for the acquireFiles method of the Subscription Class

        """
        #Cleaning possible files already occupying the available set
        self.dummySubscription.acquireFiles()

        # First test - Test if initial file (on available set) is inserted in the
        # acquired set - no arguments

        dummyFile2 = File('/tmp/dummyfile2,8888',1,1,1)
        #Insert dummyFile2 into the available files Set at dummySubscription
        self.dummySubscription.available.addFile(dummyFile2)

        S = self.dummySubscription.available.listNewFiles()
        #Check if Set returned by method is the same that was at the previous
        #available FileSet
        assert S == self.dummySubscription.acquireFiles(), \
        'Couldn\'t acquire file using method acquireFiles - (no arguments test)'


        #Second test - Test if target files are inserted at the acquired set

        dummyFileList = set()
        #Populating the dummy List with a random number of files
        for i in range(1, random.randint(100,1000)):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn = lfn, size = size, events = events,
                        checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            dummyFileList.add(file)

        #Check if return value is correct - with parameters
        acqFiles = self.dummySubscription.acquireFiles(files = dummyFileList)
        assert acqFiles == dummyFileList,\
                'Return value for acquireFiles method not the acquired files'
        #Check if all files were inserted at subscription acquired files Set
        for x in dummyFileList:
            assert x in self.dummySubscription.acquired.getFiles(type='set'), \
            'Couldn\'t acquire File %s' % x.dict['lfn']
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import listNewFiles [as 别名]
class Subscription(Pickleable, dict):
    def __init__(self, fileset = None, workflow = None,
                 split_algo = "FileBased", type = "Processing"):
        if fileset == None:
            fileset = Fileset()

        self.setdefault('fileset', fileset)
        self.setdefault('workflow', workflow)
        self.setdefault('type', type)

        self.setdefault('split_algo', split_algo)

        self.available = Fileset(name=fileset.name,
                                 files = fileset.getFiles())

        self.acquired = Fileset(name='acquired')
        self.completed = Fileset(name='completed')
        self.failed = Fileset(name='failed')

    def name(self):
        return self.getWorkflow().name.replace(' ', '') + '_' + \
                    self.getFileset().name.replace(' ', '')

    def getWorkflow(self):
        return self["workflow"]

    def workflowName(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].name

    def workflowType(self):
        if self["workflow"] == None:
            return "Unknown"
        return self["workflow"].wfType    

    def taskName(self):
        if self['workflow'] == None:
            return "Unknown"
        return self['workflow'].task

    def owner(self):
        if self['workflow'] == None:
            return 'Unknown'
        return self['workflow'].owner

    def getFileset(self):
        return self['fileset']

    def acquireFiles(self, files = [], size=1):
        """
        Return the files acquired
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        retval = []
        if len(files):
            for i in files:
                # Check each set, instead of elif, just in case something has
                # got out of synch
                if i in self.available.files:
                    self.available.files.remove(i)
                if i in self.failed.files:
                    self.failed.files.remove(i)
                if i in self.completed.files:
                    self.completed.files.remove(i)
                self.acquired.addFile(i)
        else:
            if len(self.available.files) < size or size == 0:
                size = len(self.available.files)
            for i in range(size):
                self.acquired.addFile(self.available.files.pop())

        return self.acquired.listNewFiles()

    def completeFiles(self, files):
        """
        Return the number of files complete
        """
        self.acquired.commit()
        self.available.commit()
        self.failed.commit()
        self.completed.commit()
        for i in files:
            # Check each set, instead of elif, just in case something has
            # got out of synch
            if i in self.available.files:
                self.available.files.remove(i)
            if i in self.failed.files:
                self.failed.files.remove(i)
            if i in self.acquired.files:
                self.acquired.files.remove(i)
            self.completed.addFile(i)

    def failFiles(self, files):
        """
        Return the number of files failed
        """
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.DataStructs.Fileset import Fileset [as 别名]
# 或者: from WMCore.DataStructs.Fileset.Fileset import listNewFiles [as 别名]
class FilesetTest (unittest.TestCase):
    """ 
    _FilesetTest_

    Testcase for Fileset
    """
    def setUp(self):
        """
        Create a dummy fileset and populate it with random files,
        in order to use it for the testcase methods
        
        """
        logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename=__file__.replace('.py','.log'),
                    filemode='w')
        self.logger = logging.getLogger('FilesetClassTest')
        
        #Setup the initial testcase environment:
        initialfile = File('/tmp/lfn1',1000,1,1,1)
        self.initialSet = set()
        self.initialSet.add(initialfile)
        
        #Create a Fileset, containing a initial file on it.
        self.fileset = Fileset(name = 'self.fileset', files = self.initialSet)
        #Populate the fileset with random files
        for i in range(1,1000):
            lfn = '/store/data/%s/%s/file.root' % (random.randint(1000, 9999),
                                              random.randint(1000, 9999))
            size = random.randint(1000, 2000)
            events = 1000
            run = random.randint(0, 2000)
            lumi = random.randint(0, 8)

            file = File(lfn=lfn, size=size, events=events, checksums = {"cksum": "1"})
            file.addRun(Run(run, *[lumi]))
            self.fileset.addFile(file)
        
    def tearDown(self):
        """
            No tearDown method for this testcase
            
        """
        pass
    def testAddFile(self):
        """
            Testcase for the addFile method of the Fileset class
            
        """
        #First test - Add file and check if its there
        testfile = File('/tmp/lfntest',9999,9,9)
        self.fileset.addFile(testfile)
        assert(testfile in self.fileset.listNewFiles(), 'Couldn\'t add file ' +
                'to fileset - fileset.addfile method not working')
        #Second test - Add file that was already at Fileset.files , 
        # and check if it gets updated
        testFileSame = File('/tmp/lfntest',9999,9,9)
        testFileSame.setLocation(set('dummyse.dummy.com'))
        self.fileset.addFile(testFileSame)
        assert(testFileSame in  self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile not updating location of already ' +
               'existing files' )
        assert(testfile in self.fileset.getFiles(),'Same file copy ' +
               'failed - fileset.addFile unable to remove previous file ' +
               'from list')
        #Third test - Add file that was already at Fileset.newfiles , 
        #and check if it gets updated
        assert(testFileSame in  self.fileset.listNewFiles(),'Same file copy ' +
               'failed - fileset.addFile not adding file to fileset.newFiles')
        
    def testListFiles(self):
        """
        _testListFiles_
        
        Testcase for the getFiles() method of the Fileset class
        """
        filesettemp = self.fileset.getFiles()
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetFiles(self):
        """
        Check that all files returned by the set are the same as those added to 
        the fileset
        """
        filesettemp = self.fileset.getFiles(type = "set")
        for x in filesettemp:
            assert x in (self.fileset.files | self.fileset.newfiles), \
            'Missing file %s from file list returned from fileset.ListFiles' % x["lfn"]
            
    def testSetListCompare(self):
        """
        Test that all files in fileset.setFiles are in fileset.getFiles()
        """
        thelist = self.fileset.getFiles()
        theset = self.fileset.getFiles(type = "set")
        for x in thelist:
            assert x in (theset), \
#.........这里部分代码省略.........