Python LumiList.getCompactList方法代码示例

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def testRuns(self):
        """
        Test constucting from run and list of lumis
        """
        runsAndLumis = {
            1: range(1, 34) + [35] + range(37, 48),
            2: range(49, 76) + range(77, 131) + range(133, 137)
        }
        runsAndLumis2 = {
            '1': range(1, 34) + [35] + range(37, 48),
            '2': range(49, 76) + range(77, 131) + range(133, 137)
        }
        blank = {
            '1': [],
            '2': []
        }

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList   = jsonLister.getCompactList()

        runLister = LumiList(runsAndLumis = runsAndLumis)
        runString = runLister.getCMSSWString()
        runList   = runLister.getCompactList()

        runLister2 = LumiList(runsAndLumis = runsAndLumis2)
        runList2 = runLister2.getCompactList()

        runLister3 = LumiList(runsAndLumis = blank)


        self.assertTrue(jsonString == runString)
        self.assertTrue(jsonList   == runList)
        self.assertTrue(runList2   == runList)
        self.assertTrue(len(runLister3) == 0)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
 def mergeLumis(inputdata, lumimask):
     """
     Computes the processed lumis, merges if needed and returns the compacted list (called when usedbs=no).
     """
     mergedLumis = set()
     #merge the lumis from single files
     for reports in inputdata.values():
         for report in reports:
             for run, lumis in literal_eval(report['runlumi']).iteritems():
                 for lumi in lumis:
                     mergedLumis.add((run,int(lumi))) #lumi is str, but need int
     mergedLumis = LumiList(lumis=mergedLumis)
     diff = LumiList(compactList=lumimask) - mergedLumis
     return mergedLumis.getCompactList(), diff.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def mergeLumis(inputdata, lumimask):
        """
        Computes the processed lumis, merges if needed and returns the compacted list (called when usedbs=no).
        """
        doubleLumis = set()
        mergedLumis = set()

        #merge the lumis from single files
        for reports in inputdata.values():
            for report in reports:
                for run, lumis in literal_eval(report['runlumi']).iteritems():
                    for lumi in lumis:
                        if (run,lumi) in mergedLumis:
                            doubleLumis.add((run,lumi))
                        mergedLumis.add((run,lumi))

        #convert the runlumis from list of pairs to dict: [(123,3), (123,4), (123,5), (123,7), (234,6)] => {123 : [3,4,5,7], 234 : [6]}
        dLumisDict = {}
        mLumisDict = {}
        for k, v in doubleLumis:
            dLumisDict.setdefault(k, []).append(int(v))
        for k, v in mergedLumis:
            mLumisDict.setdefault(k, []).append(int(v))

        doubleLumis = LumiList(runsAndLumis=dLumisDict)
        mergedLumis = LumiList(runsAndLumis=mLumisDict)

        #get the compact list using CMSSW framework
        return mergedLumis.getCompactList(), (LumiList(compactList=lumimask) - mergedLumis).getCompactList(), doubleLumis.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
def fast_getDoubleLumis(lumisDict):
    doubleLumis = set()
    for run, lumis in lumisDict.iteritems():
        seen = set()
        doubleLumis.update(set((run, lumi) for lumi in lumis if (run, lumi) in seen or seen.add((run, lumi))))
    doubleLumis = LumiList(lumis=doubleLumis)
    return doubleLumis.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
 def getDoubleLumis(lumisDict):
     #calculate lumis counted twice
     doubleLumis = set()
     for run, lumis in lumisDict.iteritems():
         seen = set()
         doubleLumis.update(set((run, lumi) for lumi in lumis if (run, lumi) in seen or seen.add((run, lumi))))
     doubleLumis = LumiList(lumis=doubleLumis)
     return doubleLumis.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
def makeLumiList(lumiDict):
    try:
        if isinstance(lumiDict, basestring):
            lumiDict = JsonWrapper.loads(lumiDict)
        ll = LumiList(compactList=lumiDict)
        return ll.getCompactList()
    except:
        raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
 def subtractLumis(input, output):
     """
     Computes the processed lumis, merges from the DBS reuslts (called when usedbs=yes).
     """
     out = LumiList(runsAndLumis=output)
     in_ = LumiList(runsAndLumis=input)
     diff = in_ - out
     return out.getCompactList(), diff.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
 def getDoubleLumis(lumisDict):
     #calculate lumis counted twice
     doubleLumis = set()
     for run, lumis in lumisDict.iteritems():
         for lumi in lumis:
             if lumisDict[run].count(lumi) > 1:
                 doubleLumis.add((run,lumi))
     doubleLumis = LumiList(lumis=doubleLumis)
     return doubleLumis.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def testNull(self):
        """
        Test a null list
        """

        runLister = LumiList(lumis = None)

        self.assertTrue(runLister.getCMSSWString() == '')
        self.assertTrue(runLister.getLumis() == [])
        self.assertTrue(runLister.getCompactList() == {})

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def removeLumiList(self, lumiList):
        """
        Remove a lumi list from this data structure

        This requires conversion to LumiList to do the lumi algebra an
        may be computationally expensive for a large number of lumis.
        """
        myLumis = LumiList(compactList=self['runAndLumis'])
        myLumis = myLumis - lumiList
        self['runAndLumis'] = myLumis.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def testList(self):
        """
        Test constucting from list of pairs
        """

        listLs1 = range(1, 34) + [35] + range(37, 48)
        listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
        lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)

        jsonLister = LumiList(filename = 'lumiTest.json')
        jsonString = jsonLister.getCMSSWString()
        jsonList = jsonLister.getCompactList()

        pairLister = LumiList(lumis = lumis)
        pairString = pairLister.getCMSSWString()
        pairList = pairLister.getCompactList()

        self.assertTrue(jsonString == pairString)
        self.assertTrue(jsonList   == pairList)

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def subtractLumis(input, output):
        """
        Computes the processed lumis, merges from the DBS reuslts (called when usedbs=yes).
        """
        out = LumiList(runsAndLumis=output)
        in_ = LumiList(runsAndLumis=input)
        diff = in_ - out

        #calculate lumis counted twice
        doubleLumis = set()
        for run,lumis in output.iteritems():
            for lumi in lumis:
                if output[run].count(lumi) > 1:
                    doubleLumis.add((run,lumi))
        dLumisDict = {}
        for k, v in doubleLumis:
            dLumisDict.setdefault(k, []).append(v)
        double = LumiList(runsAndLumis=dLumisDict)

        return out.getCompactList(), diff.getCompactList(), double.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def adjustLumisForCompletion(self, task, unprocessed):
        """Sets the run, lumi information in the task information for the
        completion jobs.  Returns True if completion jobs are needed,
        otherwise False.
        """
        missingDir = "automatic_splitting/missing_lumis/" #TODO in ServerUtilities to be shared with PJ

        try:
            available = set(os.listdir(missingDir)) & unprocessed
        except OSError:
            available = set()

        failed = set(self.failedJobs) & unprocessed

        if len(available) == 0 and len(failed) == 0:
            return False

        missing = LumiList()
        for missingFile in available:
            with open(os.path.join(missingDir, missingFile)) as fd:
                self.logger.info("Adding missing lumis from job %s", missingFile)
                missing = missing + LumiList(compactList=literal_eval(fd.read()))
        for failedId in failed:
            f = None
            try:
                tmpdir = tempfile.mkdtemp()
                f = tarfile.open("run_and_lumis.tar.gz")
                fn = "job_lumis_{0}.json".format(failedId)
                f.extract(fn, path=tmpdir)
                with open(os.path.join(tmpdir, fn)) as fd:
                    injson = json.load(fd)
                    missing = missing + LumiList(compactList=injson)
                    self.logger.info("Adding lumis from failed job %s", failedId)
            finally:
                if f:
                    f.close()
                shutil.rmtree(tmpdir)
        missing_compact = missing.getCompactList()
        runs = missing.getRuns()
        # Compact list is like
        # {
        # '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
        # '2':[[1,45],[50,80]]
        # }
        # Now we turn lumis it into something like:
        # lumis=['1, 33, 35, 35, 37, 47, 49, 75, 77, 130, 133, 136','1,45,50,80']
        # which is the format expected by buildLumiMask in the splitting algorithm
        lumis = [",".join(str(l) for l in functools.reduce(lambda x, y:x + y, missing_compact[run])) for run in runs]

        task['tm_split_args']['runs'] = runs
        task['tm_split_args']['lumis'] = lumis

        return True

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def adjust(self, parameters, inputs, outputs, se):
        local = self._local
        if local and se.transfer_inputs():
            inputs += [(se.local(f), os.path.basename(f), False) for id, f in self._files if f]
        if se.transfer_outputs():
            outputs += [(se.local(rf), os.path.basename(lf)) for lf, rf in self.outputs]

        parameters['mask']['files'] = self.input_files
        parameters['output files'] = self.outputs
        if not self._file_based:
            ls = LumiList(lumis=set([(run, lumi) for (id, file, run, lumi) in self._units]))
            parameters['mask']['lumis'] = ls.getCompactList()

# 需要导入模块: from WMCore.DataStructs.LumiList import LumiList [as 别名]
# 或者: from WMCore.DataStructs.LumiList.LumiList import getCompactList [as 别名]
    def notestRead(self):
        """
        Test reading from JSON
        """
        exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
        exDict = {"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}
        exVLBR = cms.VLuminosityBlockRange("1:1-1:33", "1:35", "1:37-1:47", "2:49-2:75", "2:77-2:130", "2:133-2:136")

        jsonList = LumiList(filename="lumiTest.json")
        lumiString = jsonList.getCMSSWString()
        lumiList = jsonList.getCompactList()
        lumiVLBR = jsonList.getVLuminosityBlockRange(True)

        self.assertTrue(lumiString == exString)
        self.assertTrue(lumiList == exDict)
        self.assertTrue(lumiVLBR == exVLBR)