Python LumiList.getCMSSWString方法代码示例

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testRuns(self):
        """
        Test constucting from run and list of lumis
        """
        runsAndLumis = {
            1: range(1, 34) + [35] + range(37, 48),
            2: range(49, 76) + range(77, 131) + range(133, 137)
        }
        runsAndLumis2 = {
            '1': range(1, 34) + [35] + range(37, 48),
            '2': range(49, 76) + range(77, 131) + range(133, 137)
        }
        blank = {
            '1': [],
            '2': []
        }

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList   = jsonLister.getCompactList()

        runLister = LumiList(runsAndLumis = runsAndLumis)
        runString = runLister.getCMSSWString()
        runList   = runLister.getCompactList()

        runLister2 = LumiList(runsAndLumis = runsAndLumis2)
        runList2 = runLister2.getCompactList()

        runLister3 = LumiList(runsAndLumis = blank)


        self.assertTrue(jsonString == runString)
        self.assertTrue(jsonList   == runList)
        self.assertTrue(runList2   == runList)
        self.assertTrue(len(runLister3) == 0)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testRemoveSelect(self):
        """
        a-b for lots of cases
        """

        alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
                  '2' : range(6,20) + range (30,40),
                  '3' : range(10,20) + range (30,40) + range(50,60),
                  '4' : range(10,20) + range (30,80),
                 }

        result = {'2' : range(6,20) + range (30,40),
                  '4' : range(10,20) + range (30,80),
                 }

        rem = LumiList(runsAndLumis = alumis)
        sel = LumiList(runsAndLumis = alumis)
        res = LumiList(runsAndLumis = result)

        rem.removeRuns([1,3])
        sel.selectRuns([2,4])

        self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
        self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
        self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testOr(self):
        """
        a|b for lots of cases
        """

        alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
                  '2' : range(6,20) + range (30,40),
                  '3' : range(10,20) + range (30,40) + range(50,60),
                 }
        blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
                  '2' : range(10,35),
                  '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
                 }
        clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
                  '2' : range(10,35),
                 }
        result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
                  '2' : range(6,20) + range (30,40) + range(10,35),
                  '3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70),
                 }
        a = LumiList(runsAndLumis = alumis)
        b = LumiList(runsAndLumis = blumis)
        c = LumiList(runsAndLumis = blumis)
        r = LumiList(runsAndLumis = result)
        self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString())
        self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
        self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())

        # Test list constuction (faster)

        multiple = [alumis, blumis, clumis]
        easy = LumiList(runsAndLumis = multiple)
        hard = a + b
        hard += c
        self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testAnd(self):
        """
        a&b for lots of cases
        """

        alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
                  '2' : range(6,20) + range (30,40),
                  '3' : range(10,20) + range (30,40) + range(50,60),
                  '4' : range(1,100),
                 }
        blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36),
                  '2' : range(10,35),
                  '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
                  '5' : range(1,100),
                 }
        result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49),
                  '2' : range(10,20) + range(30,35),
                  '3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60),
                 }
        a = LumiList(runsAndLumis = alumis)
        b = LumiList(runsAndLumis = blumis)
        r = LumiList(runsAndLumis = result)
        self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString())
        self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
        self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testList(self):
        """
        Test constucting from list of pairs
        """

        listLs1 = range(1, 34) + [35] + range(37, 48)
        listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
        lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList = jsonLister.getCompactList()

        pairLister = LumiList(lumis = lumis)
        pairString = pairLister.getCMSSWString()
        pairList = pairLister.getCompactList()

        self.assertTrue(jsonString == pairString)
        self.assertTrue(jsonList   == pairList)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testNull(self):
        """
        Test a null list
        """

        runLister = LumiList(lumis = None)

        self.assertTrue(runLister.getCMSSWString() == '')
        self.assertTrue(runLister.getLumis() == [])
        self.assertTrue(runLister.getCompactList() == {})

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testAddLumiMask(self):
        """
        _testAddLumiMask_

        Verify that setting and getting the lumiMask objects for a task works correctly.
        Do a round trip of a typical lumi mask
        """
        testTask = makeWMTask("TestTask")

        lumiMask = LumiList(compactList = {
                '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
                '2':[[1,45]],
                '3':[[1,45],[50,80]],
            })

        testTask.setLumiMask(lumiMask = lumiMask.getCompactList())
        outMask =  LumiList(compactList = testTask.getLumiMask())
        self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString())

        return

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testSubtract(self):
        """
        a-b for lots of cases
        """

        alumis = {
            "1": range(2, 20) + range(31, 39) + range(45, 49),
            "2": range(6, 20) + range(30, 40),
            "3": range(10, 20) + range(30, 40) + range(50, 60),
        }
        blumis = {
            "1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36),
            "2": range(10, 35),
            "3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70),
        }
        clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), "2": range(10, 35)}
        result = {
            "1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
            "2": range(6, 10) + range(35, 40),
            "3": range(15, 20) + range(30, 35) + range(51, 59),
        }
        result2 = {
            "1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
            "2": range(6, 10) + range(35, 40),
            "3": range(10, 20) + range(30, 40) + range(50, 60),
        }
        a = LumiList(runsAndLumis=alumis)
        b = LumiList(runsAndLumis=blumis)
        c = LumiList(runsAndLumis=clumis)
        r = LumiList(runsAndLumis=result)
        r2 = LumiList(runsAndLumis=result2)

        self.assertTrue((a - b).getCMSSWString() == r.getCMSSWString())
        self.assertTrue((a - b).getCMSSWString() != (b - a).getCMSSWString())
        # Test where c is missing runs from a
        self.assertTrue((a - c).getCMSSWString() == r2.getCMSSWString())
        self.assertTrue((a - c).getCMSSWString() != (c - a).getCMSSWString())
        # Test empty lists
        self.assertTrue(str(a - a) == "{}")
        self.assertTrue(len(a - a) == 0)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testWMagentFormat(self):
        """
        Test the constuction of the double-string-based version of lumi lists
        used in some places by WMAgent
        """

        clumis = {'1': range(2, 20) + range(31, 39) + range(45, 49),
                  '2': range(6, 20) + range(30, 40),
                  '3': range(10, 20) + range(30, 40) + range(50, 60),
                  '4': range(1, 100),
                  }
        runs = [1, 2, 3, 4]
        lumis = ['2,19,31,38,45,48', '6,19,30,39', '10,19,30,39,50,59', '1,99']

        c = LumiList(runsAndLumis=clumis)
        w = LumiList(wmagentFormat=(runs, lumis))
        self.assertEqual(w.getCMSSWString(), c.getCMSSWString())

        clumis = {'12': [4, 5]}
        runs = [12]
        lumis = ['4,5']

        c = LumiList(runsAndLumis=clumis)
        w = LumiList(wmagentFormat=(runs, lumis))
        self.assertEqual(w.getCMSSWString(), c.getCMSSWString())

        aruns = [1]
        alumis = ['1,10,11,20']
        bruns = [1]
        blumis = ['1,20']
        self.assertEqual(LumiList(wmagentFormat=(aruns, alumis)).getCMSSWString(),
                         LumiList(wmagentFormat=(bruns, blumis)).getCMSSWString())

        with self.assertRaises(RuntimeError):
            w = LumiList(wmagentFormat=([1,2,3]))  # No lumis
        with self.assertRaises(RuntimeError):
            w = LumiList(wmagentFormat=([1], ['1,2,3']))  # Need twice as many lumis as runs

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def notestRead(self):
        """
        Test reading from JSON
        """
        exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
        exDict = {"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}
        exVLBR = cms.VLuminosityBlockRange("1:1-1:33", "1:35", "1:37-1:47", "2:49-2:75", "2:77-2:130", "2:133-2:136")

        jsonList = LumiList(filename="lumiTest.json")
        lumiString = jsonList.getCMSSWString()
        lumiList = jsonList.getCompactList()
        lumiVLBR = jsonList.getVLuminosityBlockRange(True)

        self.assertTrue(lumiString == exString)
        self.assertTrue(lumiList == exDict)
        self.assertTrue(lumiVLBR == exVLBR)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testOr(self):
        """
        a|b for lots of cases
        """

        alumis = {
            "1": range(2, 20) + range(31, 39) + range(45, 49),
            "2": range(6, 20) + range(30, 40),
            "3": range(10, 20) + range(30, 40) + range(50, 60),
        }
        blumis = {
            "1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80),
            "2": range(10, 35),
            "3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70),
        }
        clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80), "2": range(10, 35)}
        result = {
            "1": range(2, 20)
            + range(31, 39)
            + range(45, 49)
            + range(1, 6)
            + range(12, 13)
            + range(16, 30)
            + range(40, 50)
            + range(39, 80),
            "2": range(6, 20) + range(30, 40) + range(10, 35),
            "3": range(10, 20)
            + range(30, 40)
            + range(50, 60)
            + range(10, 15)
            + range(35, 40)
            + range(45, 51)
            + range(59, 70),
        }
        a = LumiList(runsAndLumis=alumis)
        b = LumiList(runsAndLumis=blumis)
        c = LumiList(runsAndLumis=blumis)
        r = LumiList(runsAndLumis=result)
        self.assertTrue((a | b).getCMSSWString() == r.getCMSSWString())
        self.assertTrue((a | b).getCMSSWString() == (b | a).getCMSSWString())
        self.assertTrue((a | b).getCMSSWString() == (a + b).getCMSSWString())

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testLumiMask(self):
        """
        _testLumiMask_

        Test that we can use a lumi-mask to filter good runs/lumis.
        """
        splitter = SplitterFactory()

        # Create 3 files with 100 events per lumi:
        # - file1 with 1 run  of 8 lumis
        # - file2 with 2 runs of 2 lumis each
        # - file3 with 1 run  of 5 lumis
        fileA = File(lfn="/this/is/file1", size=1000, events=800)
        fileB = File(lfn="/this/is/file2", size=1000, events=400)
        fileC = File(lfn="/this/is/file3", size=1000, events=500)

        lumiListA = []
        for lumi in range(8):
            lumiListA.append(10 + lumi)
        fileA.addRun(Run(1, *lumiListA))
        fileA.setLocation("somese.cern.ch")
        lumiListB1 = []
        lumiListB2 = []
        for lumi in range(2):
            lumiListB1.append(20 + lumi)
            lumiListB2.append(30 + lumi)
        fileB.addRun(Run(2, *lumiListB1))
        fileB.addRun(Run(3, *lumiListB2))
        fileB.setLocation("somese.cern.ch")
        lumiListC = []
        for lumi in range(5):
            lumiListC.append(40 + lumi)
        fileC.addRun(Run(4, *lumiListC))
        fileC.setLocation("somese.cern.ch")

        testFileset = Fileset(name='Fileset')
        testFileset.addFile(fileA)
        testFileset.addFile(fileB)
        testFileset.addFile(fileC)

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=self.testWorkflow,
                                        split_algo="EventAwareLumiByWork",
                                        type="Processing")
        jobFactory = splitter(package="WMCore.DataStructs",
                              subscription=testSubscription)

        # Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]}
        jobGroups = jobFactory(halt_job_on_file_boundaries=False,
                               splitOnRun=False,
                               events_per_job=850,
                               runs=['1', '2', '4'],
                               lumis=['10,14', '20,21', '40,41'],
                               performance=self.performanceParams)

        self.assertEqual(len(jobGroups), 1, "There should be only one job group")
        jobs = jobGroups[0].jobs
        self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup")
        processedLumis = LumiList()
        for job in jobs:
            processedLumis += LumiList(compactList=job['mask'].getRunAndLumis())
        correctLumis = LumiList(compactList={1: [[10, 14]], 2: [[20, 21]], 4: [[40, 41]]})
        self.assertEqual(processedLumis.getCMSSWString(), correctLumis.getCMSSWString())

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCMSSWString [as 别名]
    def testGetLumiWhitelist(self):
        """
        _testGetLumiWhitelist_

        Verify that the ACDC whitelist generation code works correctly.  We'll
        add jobs with the following lumi info:
          # Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12]
          # Run 2, lumis [5, 6, 7], [10, 11, 12], [15]
          # Run 3, lumis [20]

        And should get out a whitelist that looks like this:
          {"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
           "2": [[5, 7], [10, 12], [15, 15]],
           "3": [[20, 20]]}
        """
        dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc")

        testFileA = File(lfn=makeUUID(), size=1024, events=1024)
        testFileA.addRun(Run(1, 1, 2))
        testFileB = File(lfn=makeUUID(), size=1024, events=1024)
        testFileB.addRun(Run(1, 3))
        testJobA = self.getMinimalJob()
        testJobA.addFile(testFileA)
        testJobA.addFile(testFileB)

        testFileC = File(lfn=makeUUID(), size=1024, events=1024)
        testFileC.addRun(Run(1, 4, 6))
        testJobB = self.getMinimalJob()
        testJobB.addFile(testFileC)

        testFileD = File(lfn=makeUUID(), size=1024, events=1024)
        testFileD.addRun(Run(1, 7))
        testJobC = self.getMinimalJob()
        testJobC.addFile(testFileD)

        testFileE = File(lfn=makeUUID(), size=1024, events=1024)
        testFileE.addRun(Run(1, 11, 12))
        testJobD = self.getMinimalJob()
        testJobD.addFile(testFileE)

        testFileF = File(lfn=makeUUID(), size=1024, events=1024)
        testFileF.addRun(Run(2, 5, 6, 7))
        testJobE = self.getMinimalJob()
        testJobE.addFile(testFileF)

        testFileG = File(lfn=makeUUID(), size=1024, events=1024)
        testFileG.addRun(Run(2, 10, 11, 12))
        testJobF = self.getMinimalJob()
        testJobF.addFile(testFileG)

        testFileH = File(lfn=makeUUID(), size=1024, events=1024)
        testFileH.addRun(Run(2, 15))
        testJobG = self.getMinimalJob()
        testJobG.addFile(testFileH)

        testFileI = File(lfn=makeUUID(), size=1024, events=1024)
        testFileI.addRun(Run(3, 20))
        testJobH = self.getMinimalJob()
        testJobH.addFile(testFileI)

        testFileJ = File(lfn=makeUUID(), size=1024, events=1024)
        testFileJ.addRun(Run(1, 9))
        testJobI = self.getMinimalJob()
        testJobI.addFile(testFileJ)

        dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE,
                        testJobF, testJobG, testJobH, testJobI])
        whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco")

        self.assertEqual(len(whiteList.keys()), 3,
                         "Error: There should be 3 runs.")
        self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]],
                         "Error: Whitelist for run 1 is wrong.")
        self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]],
                         "Error: Whitelist for run 2 is wrong.")
        self.assertEqual(whiteList["3"], [[20, 20]],
                         "Error: Whitelist for run 3 is wrong.")

        correctLumiList = LumiList(compactList={"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
                                                "2": [[5, 7], [10, 12], [15, 15]],
                                                "3": [[20, 20]]})
        testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco")
        self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString())

        return