Python LumiList.getRuns方法代码示例

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
 def createDataDatasets(self):
   self.dataDatasets = {}
   for d in self.datasets:
     dsLumiList = None
     if not os.path.isfile(d['json']):
       oldSsArgv = sys.argv; sys.argv=[] # sys argv fix
       dasC = dasTools.myDasClient();dasC.limit=0
       dsLumiList = dasC.getJsonOfDataset(d["dataset"])
       dsLumiList.writeJSON(d['json'])
       sys.argv = oldSsArgv
     else:
       dsLumiList = LumiList(compactList=json.load(open(d['json'])))
     dsRuns = dsLumiList.getRuns()
     self.dataDatasets[d['label']] = ('{ \n '
       '\t"xSec":None\n'
       '\t,"localFile":None\n'
       '\t,"datasetName":"'+d["dataset"]+'"\n'
       '\t,"label":"Data_'+d['label']+'"\n'
       '\t,"datasetJSON":"'+d['json']+'"\n'
       '\t,"crabConfig":{\n'
         '\t\t"CMSSW":{"lumis_per_job":5\n'
           '\t\t\t,"lumi_mask": os.getenv("CMSSW_BASE") + '+'"/'+d['goldenJson'].lstrip('/')+'"\n'
           '\t\t\t,"total_number_of_lumis" : -1}\n'
         '\t\t}\n'
       '\t,"color":0\n'
       '\t,"runRange":"'+str(dsRuns[0])+"-"+str(dsRuns[-1])+'"\n'
     '\t}\n');

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
def getRuns(name=None,bfield=None,bunchSpacing=None):
	ll = LumiList()
	for rp in runPeriods:
		if name is None or rp.name == name:
			if bfield is None or rp.bfield == bfield:
				if bunchSpacing is None or rp.bunchSpacing == bunchSpacing:
					newll = LumiListForRunPeriod(rp)
					ll += LumiListForRunPeriod(rp)
	return ll.getRuns()

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
def LumiListForRunPeriod(rp, MIN_LUMIS=0):
	ll = LumiList(filename = rp.json)
	runs = [ run for run in map(int,ll.getRuns()) if run >= rp.firstRun and run <= rp.lastRun]

	lumis = ll.getLumis()
	nlumis = defaultdict(int)
	for r,l in lumis:
		nlumis[r]+=1
	select_runs = [run for run in runs if nlumis[run] > MIN_LUMIS]
	ll.selectRuns(select_runs)
	return ll

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
def shortenJson(jsonFile,minRun=0,maxRun=-1,output=None,debug=False):
  from copy import deepcopy
  runList = jsonFile 
  if isinstance(runList,LumiList):
    runList = deepcopy(jsonFile)
  else:
    runList = LumiList (filename = jsonFile)  # Read in first  JSON file
  allRuns = runList.getRuns()
  runsToRemove=[]
  for run in allRuns:
      if  int(run) < minRun:
          runsToRemove.append (run)
      if maxRun > 0 and int(run) > maxRun:
          runsToRemove.append (run)
  if debug:
	print " runsToRemove ",runsToRemove
  runList.removeRuns (runsToRemove)
  if output:
    runList.writeJSON (output)
  else:
    return  runList

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
    parser.add_option ('--output', dest='output', type='string',
                       help='Save output to file OUTPUT')
    # required parameters
    (options, args) = parser.parse_args()
    if len (args) != 1:
        raise RuntimeError, "Must provide exactly one input file"

    if options.min and options.max and options.min > options.max:
        raise RuntimeError, "Minimum value (%d) is greater than maximum value (%d)" % (options.min, options.max)

    commaRE = re.compile (r',')
    runsToRemove = []
    for chunk in options.runs:
        runs = commaRE.split (chunk)
        runsToRemove.extend (runs)

    alphaList = LumiList (filename = args[0]) # Read in first JSON file
    allRuns = alphaList.getRuns()
    for run in allRuns:
        if options.min and int(run) < options.min:
            runsToRemove.append (run)
        if options.max and int(run) > options.max:
            runsToRemove.append (run)

    alphaList.removeRuns (runsToRemove)

    if options.output:
        alphaList.writeJSON (options.output)
    else:
        print alphaList

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
    def __lumiSelectionSnippet( self, jsonPath = None, firstRun = None, lastRun = None ):
        lumiSecExtend = ""
        if firstRun or lastRun or jsonPath:
            if not jsonPath:
                selectedRunList = self.__getRunList()
                if firstRun:
                    selectedRunList = [ run for run in selectedRunList \
                                        if self.__findInJson(run, "run_number") >= firstRun ]
                if lastRun:
                    selectedRunList = [ run for run in selectedRunList \
                                        if self.__findInJson(run, "run_number") <= lastRun ]
                lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
                             + str( self.__findInJson(run, "run_number") ) + ":max" \
                             for run in selectedRunList ]
                splitLumiList = list( self.__chunks( lumiList, 255 ) )
            else:
                theLumiList = None
                try:
                    theLumiList = LumiList ( filename = jsonPath )
                except ValueError:
                    pass

                if theLumiList is not None:
                    allRuns = theLumiList.getRuns()
                    runsToRemove = []
                    for run in allRuns:
                        if firstRun and int( run ) < firstRun:
                            runsToRemove.append( run )
                        if lastRun and int( run ) > lastRun:
                            runsToRemove.append( run )
                    theLumiList.removeRuns( runsToRemove )
                    splitLumiList = list( self.__chunks(
                        theLumiList.getCMSSWString().split(','), 255 ) )
                    if not (splitLumiList and splitLumiList[0] and splitLumiList[0][0]):
                        splitLumiList = None
                else:
                    with open(jsonPath) as f:
                        jsoncontents = f.read()
                        if "process.source.lumisToProcess" in jsoncontents:
                            msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet.  Trying to use it" % jsonPath
                            if firstRun or lastRun:
                                msg += ("\n  (after applying firstRun and/or lastRun)")
                            msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
                            msg += "\nCheck your config file to make sure that it worked properly."
                            print msg

                            runlist = self.__getRunList()
                            if firstRun or lastRun:
                                self.__firstusedrun = -1
                                self.__lastusedrun = -1
                                jsoncontents = re.sub(r"\d+:(\d+|max)(-\d+:(\d+|max))?", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
                                jsoncontents = (jsoncontents.replace("'',\n","").replace("''\n","")
                                                            .replace('"",\n','').replace('""\n',''))
                                self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
                                self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
                                if self.__lastusedrun < self.__firstusedrun:
                                    jsoncontents = None
                            else:
                                self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
                                self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
                            lumiSecExtend = jsoncontents
                            splitLumiList = None
                        else:
                            raise AllInOneError("%s is not a valid json file!" % jsonPath)

            if splitLumiList and splitLumiList[0] and splitLumiList[0][0]:
                lumiSecStr = [ "',\n'".join( lumis ) \
                               for lumis in splitLumiList ]
                lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
                               for lumis in lumiSecStr ]
                lumiSecExtend = "\n".join( lumiSecStr )
                runlist = self.__getRunList()
                self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
                self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
            elif lumiSecExtend:
                pass
            else:
                msg = "You are trying to run a validation without any runs!  Check that:"
                if firstRun or lastRun:
                    msg += "\n - firstRun/begin and lastRun/end are correct for this dataset, and there are runs in between containing data"
                if jsonPath:
                    msg += "\n - your JSON file is correct for this dataset, and the runs contain data"
                if (firstRun or lastRun) and jsonPath:
                    msg += "\n - firstRun/begin and lastRun/end are consistent with your JSON file"
                raise AllInOneError(msg)

        else:
            if self.__inputMagneticField is not None:
                pass  #never need self.__firstusedrun or self.__lastusedrun
            else:
                runlist = self.__getRunList()
                self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number"))
                self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number"))

        return lumiSecExtend

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
 def __createSnippet( self, jsonPath = None, begin = None, end = None,
                      firstRun = None, lastRun = None, repMap = None,
                      crab = False ):
     if firstRun:
         firstRun = int( firstRun )
     if lastRun:
         lastRun = int( lastRun )
     if ( begin and firstRun ) or ( end and lastRun ):
         msg = ( "The Usage of "
                 + "'begin' & 'firstRun' " * int( bool( begin and
                                                        firstRun ) )
                 + "and " * int( bool( ( begin and firstRun ) and
                                      ( end and lastRun ) ) )
                 + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
                 + "is ambigous." )
         raise AllInOneError( msg )
     if begin or end:
         ( firstRun, lastRun ) = self.convertTimeToRun(
             begin = begin, end = end, firstRun = firstRun,
             lastRun = lastRun )
     if ( firstRun and lastRun ) and ( firstRun > lastRun ):
         msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
                 "chosen is greater than the upper time/runrange limit "
                 "('end'/'lastRun').")
         raise AllInOneError( msg )
     goodLumiSecStr = ""
     lumiStr = ""
     lumiSecExtend = ""
     if firstRun or lastRun:
         goodLumiSecStr = ( "lumiSecs = cms.untracked."
                            "VLuminosityBlockRange()\n" )
         lumiStr = "                    lumisToProcess = lumiSecs,\n"
         if not jsonPath:
             selectedRunList = self.__getRunList()
             if firstRun:
                 selectedRunList = [ run for run in selectedRunList \
                                     if run["run_number"] >= firstRun ]
             if lastRun:
                 selectedRunList = [ run for run in selectedRunList \
                                     if run["run_number"] <= lastRun ]
             lumiList = [ str( run["run_number"] ) + ":1-" \
                          + str( run["run_number"] ) + ":max" \
                          for run in selectedRunList ]
             splitLumiList = list( self.__chunks( lumiList, 255 ) )
         else:
             theLumiList = LumiList ( filename = jsonPath )
             allRuns = theLumiList.getRuns()
             runsToRemove = []
             for run in allRuns:
                 if firstRun and int( run ) < firstRun:
                     runsToRemove.append( run )
                 if lastRun and int( run ) > lastRun:
                     runsToRemove.append( run )
             theLumiList.removeRuns( runsToRemove )
             splitLumiList = list( self.__chunks(
                 theLumiList.getCMSSWString().split(','), 255 ) )
         if not len(splitLumiList[0][0]) == 0:
             lumiSecStr = [ "',\n'".join( lumis ) \
                            for lumis in splitLumiList ]
             lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
                            for lumis in lumiSecStr ]
             lumiSecExtend = "\n".join( lumiSecStr )
     elif jsonPath:
             goodLumiSecStr = ( "goodLumiSecs = LumiList.LumiList(filename"
                                "= '%(json)s').getCMSSWString().split(',')\n"
                                "lumiSecs = cms.untracked"
                                ".VLuminosityBlockRange()\n"
                                )
             lumiStr = "                    lumisToProcess = lumiSecs,\n"
             lumiSecExtend = "lumiSecs.extend(goodLumiSecs)\n"
     if crab:
         files = ""
     else:
         splitFileList = list( self.__chunks( self.fileList(), 255 ) )
         fileStr = [ "',\n'".join( files ) for files in splitFileList ]
         fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
                     for files in fileStr ]
         files = "\n".join( fileStr )
     theMap = repMap
     theMap["files"] = files
     theMap["json"] = jsonPath
     theMap["lumiStr"] = lumiStr
     theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap )
     theMap["lumiSecExtend"] = lumiSecExtend
     if crab:
         dataset_snippet = self.__dummy_source_template%( theMap )
     else:
         dataset_snippet = self.__source_template%( theMap )
     return dataset_snippet

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
    def _dasPopen(dbs):
        if 'LSB_JOBID' in os.environ:
            raise RuntimeError, "Trying to do a DAS query while in a LXBatch job (env variable LSB_JOBID defined)\nquery was: %s" % dbs
        if 'X509_USER_PROXY' in os.environ:
            dbs += " --key {0} --cert {0}".format(os.environ['X509_USER_PROXY'])
        logger.info('DAS query\t: %s',  dbs)
        return os.popen(dbs)

    dbs='das_client --query="run dataset=%s instance=prod/%s" --limit %i'%(prompt.heppy.dataset, 'global', 0)
    prompt_runs = [int(r) for r in _dasPopen(dbs).readlines()]
    dbs='das_client --query="run dataset=%s instance=prod/%s" --limit %i'%(rereco.heppy.dataset, 'global', 0)
    rereco_runs =[int(r) for r in  _dasPopen(dbs).readlines()]

    runs = []
    for str_run in lumiList.getRuns():
        run = int(str_run)
        if run in prompt_runs and run in rereco_runs:
            runs.append(run)

    print "Now running %i jobs: %r"%( len(runs), runs )

    import subprocess
    def wrapper( run_ ):
        subprocess.call(["python", "jetTreeMaker.py", ("--era=%s"%args.era), ("--run=%i"%run_) ])

    from multiprocessing import Pool
    pool = Pool( 10 )
    results = pool.map(wrapper, runs)
    pool.close()

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
    def __createSnippet( self, jsonPath = None, begin = None, end = None,
                         firstRun = None, lastRun = None, repMap = None,
                         crab = False, parent = False ):
        if firstRun:
            firstRun = int( firstRun )
        if lastRun:
            lastRun = int( lastRun )
        if ( begin and firstRun ) or ( end and lastRun ):
            msg = ( "The Usage of "
                    + "'begin' & 'firstRun' " * int( bool( begin and
                                                           firstRun ) )
                    + "and " * int( bool( ( begin and firstRun ) and
                                         ( end and lastRun ) ) )
                    + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
                    + "is ambigous." )
            raise AllInOneError( msg )
        if begin or end:
            ( firstRun, lastRun ) = self.convertTimeToRun(
                begin = begin, end = end, firstRun = firstRun,
                lastRun = lastRun )
        if ( firstRun and lastRun ) and ( firstRun > lastRun ):
            msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
                    "chosen is greater than the upper time/runrange limit "
                    "('end'/'lastRun').")
            raise AllInOneError( msg )
        if self.predefined() and (jsonPath or begin or end or firstRun or lastRun):
            msg = ( "The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun'"
                    "only work for official datasets, not predefined _cff.py files" )
            raise AllInOneError( msg )
        goodLumiSecStr = ""
        lumiStr = ""
        lumiSecExtend = ""
        if firstRun or lastRun or jsonPath:
            goodLumiSecStr = ( "lumiSecs = cms.untracked."
                               "VLuminosityBlockRange()\n" )
            lumiStr = "                    lumisToProcess = lumiSecs,\n"
            if not jsonPath:
                selectedRunList = self.__getRunList()
                if firstRun:
                    selectedRunList = [ run for run in selectedRunList \
                                        if self.__findInJson(run, "run_number") >= firstRun ]
                if lastRun:
                    selectedRunList = [ run for run in selectedRunList \
                                        if self.__findInJson(run, "run_number") <= lastRun ]
                lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
                             + str( self.__findInJson(run, "run_number") ) + ":max" \
                             for run in selectedRunList ]
                splitLumiList = list( self.__chunks( lumiList, 255 ) )
            else:
                theLumiList = None
                try:
                    theLumiList = LumiList ( filename = jsonPath )
                except ValueError:
                    pass

                if theLumiList is not None:
                    allRuns = theLumiList.getRuns()
                    runsToRemove = []
                    for run in allRuns:
                        if firstRun and int( run ) < firstRun:
                            runsToRemove.append( run )
                        if lastRun and int( run ) > lastRun:
                            runsToRemove.append( run )
                    theLumiList.removeRuns( runsToRemove )
                    splitLumiList = list( self.__chunks(
                        theLumiList.getCMSSWString().split(','), 255 ) )
                else:
                    with open(jsonPath) as f:
                        jsoncontents = f.read()
                        if "process.source.lumisToProcess" in jsoncontents:
                            msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet.  Trying to use it" % jsonPath
                            if firstRun or lastRun:
                                msg += ("\n  (after applying firstRun and/or lastRun)")
                            msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
                            msg += "\nCheck your config file to make sure that it worked properly."
                            print msg

                            runlist = self.__getRunList()
                            if firstRun or lastRun:
                                self.__firstusedrun = -1
                                self.__lastusedrun = -1
                                jsoncontents = re.sub("\d+:(\d+|max)-\d+:(\d+|max)", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
                                self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
                                self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
                            else:
                                self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
                                self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
                            lumiSecExtend = jsoncontents
                            splitLumiList = [[""]]

            if not len(splitLumiList[0][0]) == 0:
                lumiSecStr = [ "',\n'".join( lumis ) \
                               for lumis in splitLumiList ]
                lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
                               for lumis in lumiSecStr ]
                lumiSecExtend = "\n".join( lumiSecStr )
                runlist = self.__getRunList()
                self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
                self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
        else:
#.........这里部分代码省略.........

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
import re, os, subprocess
from pprint import pprint
from collections import defaultdict
from FWCore.PythonUtilities.LumiList import LumiList
from RecoLuminosity.LumiDB import sessionManager, lumiCalcAPI, revisionDML
from JMTucker.Tools.general import from_pickle, to_pickle

os.system('mkdir -p prescales_temp')

def popen(cmd):
    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).communicate()[0]

ll = LumiList('prescales_temp/Cert_190456-208686_8TeV_PromptReco_Collisions12_JSON.txt')
ll_compact = ll.getCompactList()
runs = [int(i) for i in ll.getRuns()]
runs.sort()

def dump_lumibyls(runs):
    l = float(len(runs))
    for i,run in enumerate(runs):
        out_fn = 'prescales_temp/lumibyls/%i.csv' % run
        already = os.path.isfile(out_fn)
        print 'run %i (%i/%i)%s' % (run, i+1, l, ' (skipping since already dumped)' if already else '')
        if already:
            continue
        popen('lumiCalc2.py lumibyls -r %i -o %s' % (run, out_fn))

def parse_lumibyls(run):
    d = defaultdict(dict)
    for line in open('prescales_temp/lumibyls/%i.csv' % run):

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import getRuns [as 别名]
#!/usr/bin/env python

import os, re, sys
from collections import defaultdict
from itertools import combinations
from FWCore.PythonUtilities.LumiList import LumiList

# Should rewrite not to hit the db for every cfg, but just get the HLT
# key for each run and then only get cfgs for unique keys.

dcsonly_ll = LumiList(
    "/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions12/8TeV/DCSOnly/json_DCSONLY.txt"
)  # JMTBAD use DCSOnly_ll from goodlumis once sorted
runs = sorted(int(run) for run in dcsonly_ll.getRuns())
cmd = "edmConfigFromDB --cff --runNumber %i --noedsources --noes --noservices --nomodules"

path_re = re.compile(r"(HLT_Mu40_eta2p1_v\d+)")
prescaled_path_re = re.compile(r"(HLT_Mu15_eta2p1_v\d+)")

paths_and_filters = defaultdict(list)

for run in runs:
    print "run:", run,
    sys.stdout.flush()
    path = prescaled_path = filter = prescaled_filter = None
    for line in os.popen(cmd % run):
        if "cms.Path" not in line:
            continue
        filt = line.split(" + ")[-2]  # JMTBAD fragile
        mo = path_re.search(line)
        if mo is not None: