Python LumiList.filterLumis方法代码示例

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import filterLumis [as 别名]
def split_by_lumi(config, dataset_info, task_list):
    if config.has_key('lumi mask'):
        lumi_mask = LumiList(filename=config['lumi mask'])
        dataset_info.total_lumis = 0
        for file in dataset_info.files:
            dataset_info.lumis[file] = lumi_mask.filterLumis(dataset_info.lumis[file])
            dataset_info.total_lumis += len(dataset_info.lumis[file])

    lumis_per_task = config['lumis per task']
    lumis_processed = 0
    task_id = 0
    tasks = []
    files = iter(dataset_info.files)
    file = files.next()
    input_files_this_task = [file]
    task_lumis_remaining = dataset_info.lumis[file]
    while lumis_processed < dataset_info.total_lumis:
        for file in input_files_this_task:
            common_lumis = set(dataset_info.lumis[file]).intersection(set(task_lumis_remaining))
            if len(common_lumis) == 0 or len(dataset_info.lumis[file]) == 0:
                input_files_this_task.remove(file)
        while lumis_per_task <= len(task_lumis_remaining):
            task_lumis = LumiList(lumis=task_lumis_remaining[:lumis_per_task])
            task_lumis_remaining = task_lumis_remaining[lumis_per_task:]
            tasks.append((input_files_this_task, task_lumis.getVLuminosityBlockRange()))
            task_id += 1
            lumis_processed += lumis_per_task
        try:
            file = files.next()
            input_files_this_task.append(file)
            task_lumis_remaining.extend(dataset_info.lumis[file])
        except:
            lumis_per_task = len(task_lumis_remaining)

    with open(task_list, 'w') as json_file:
        json.dump(tasks, json_file)

    return len(tasks)

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import filterLumis [as 别名]

#.........这里部分代码省略.........
        if self.cfg_params.get('CMSSW.fileblocks_file') :
            msg = "CMSSW.fileblocks_file option non supported"
            raise CrabException(msg)
        fileBlocks_FileName = os.path.abspath(defaultName)

        if useDBS2 or verifyDBS23:
            #common.logger.info("looking up DBS2 ...")
            import DBSAPI.dbsApi
            import DBSAPI.dbsApiException
            start_time=time.time()
            api2 = DBSAPI.dbsApi.DbsApi(args)
            files2 = self.queryDbs(api2,path=self.datasetPath,runselection=runselection,useParent=useparent)
            elapsed=time.time() - start_time
            common.logger.info("DBS2 lookup took %5.2f sec" % elapsed)
            if useDBS2:
                self.files = files2
        if useDBS3 or verifyDBS23:
            #common.logger.info("looking up DBS3 ...")
            from dbs.apis.dbsClient import DbsApi
            start_time=time.time()
            api3 = DbsApi(dbs3_url)
            files3 = self.queryDbs3(api3,path=self.datasetPath,runselection=runselection,useParent=useparent)
            elapsed=time.time() - start_time
            common.logger.info("DBS3 lookup took %5.2f sec" % elapsed)
            if useDBS3:
                self.files = files3


        # Check to see what the dataset is
        pdsName = self.datasetPath.split("/")[1]
        if useDBS2 :
            primDSs = api2.listPrimaryDatasets(pdsName)
            dataType = primDSs[0]['Type']
        elif useDBS3 :
            dataType=api3.listDataTypes(dataset=self.datasetPath)[0]['data_type']

        common.logger.info("Datatype is %s" % dataType)
        if dataType == 'data' and not \
            (self.splitByRun or self.splitByLumi or self.splitDataByEvent):
            msg = 'Data must be split by lumi or by run. ' \
                  'Please see crab -help for the correct settings'
            raise  CrabException(msg)



        anFileBlocks = []
        if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)

        # parse files and fill arrays
        for file in self.files :
            parList  = []
            fileLumis = [] # List of tuples
            # skip already analyzed blocks
            fileblock = file['Block']['Name']
            if fileblock not in anFileBlocks :
                filename = file['LogicalFileName']
                # asked retry the list of parent for the given child
                if useparent==1:
                    parList = [x['LogicalFileName'] for x in file['ParentList']]
                if self.splitByLumi:
                    fileLumis = [ (x['RunNumber'], x['LumiSectionNumber'])
                                 for x in file['LumiList'] ]
                self.parent[filename] = parList
                # For LumiMask, intersection of two lists.
                if self.lumiMask and runselection:
                    self.lumis[filename] = runList.filterLumis(lumiList.filterLumis(fileLumis))
                elif runselection:
                    self.lumis[filename] = runList.filterLumis(fileLumis)
                elif self.lumiMask:
                    self.lumis[filename] = lumiList.filterLumis(fileLumis)
                else:
                    self.lumis[filename] = fileLumis

                if filename.find('.dat') < 0 :
                    events    = file['NumberOfEvents']
                    # Count number of events and lumis per block
                    if fileblock in self.eventsPerBlock.keys() :
                        self.eventsPerBlock[fileblock] += events
                    else :
                        self.eventsPerBlock[fileblock] = events
                    # Number of events per file
                    self.eventsPerFile[filename] = events

                    # List of files per block
                    if fileblock in self.blocksinfo.keys() :
                        self.blocksinfo[fileblock].append(filename)
                    else :
                        self.blocksinfo[fileblock] = [filename]

                    # total number of events
                    self.maxEvents += events
                    self.maxLumis  += len(self.lumis[filename])

        if  self.skipBlocks and len(self.eventsPerBlock.keys()) == 0:
            msg = "No new fileblocks available for dataset: "+str(self.datasetPath)
            raise  CrabException(msg)

        if len(self.eventsPerBlock) <= 0:
            msg="No data for %s in DBS\n Check datasetpath parameter in crab.cfg" % self.datasetPath
            raise  CrabException(msg)

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import filterLumis [as 别名]
    def fetchDBSInfo(self):
        """
        Contact DBS
        """
        ## get DBS URL
        global_url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
        dbs_url=  self.cfg_params.get('CMSSW.dbs_url', global_url)
        common.logger.info("Accessing DBS at: "+dbs_url)

        ## check if runs are selected
        runselection = []
        if (self.cfg_params.has_key('CMSSW.runselection')):
            runselection = parseRange2(self.cfg_params['CMSSW.runselection'])

        ## check if various lumi parameters are set
        self.lumiMask = self.cfg_params.get('CMSSW.lumi_mask',None)
        self.lumiParams = self.cfg_params.get('CMSSW.total_number_of_lumis',None) or \
                          self.cfg_params.get('CMSSW.lumis_per_job',None)

        lumiList = None
        if self.lumiMask:
            lumiList = LumiList(filename=self.lumiMask)
        if runselection:
            runList = LumiList(runs = runselection)

        self.splitByRun = int(self.cfg_params.get('CMSSW.split_by_run', 0))
        self.splitDataByEvent = int(self.cfg_params.get('CMSSW.split_by_event', 0))
        common.logger.log(10-1,"runselection is: %s"%runselection)

        if not self.splitByRun:
            self.splitByLumi = self.lumiMask or self.lumiParams or self.ads

        if self.splitByRun and not runselection:
            msg = "Error: split_by_run must be combined with a runselection"
            raise CrabException(msg)

        ## service API
        args = {}
        args['url']     = dbs_url
        args['level']   = 'CRITICAL'

        ## check if has been requested to use the parent info
        useparent = int(self.cfg_params.get('CMSSW.use_parent',0))

        ## check if has been asked for a non default file to store/read analyzed fileBlocks
        defaultName = common.work_space.shareDir()+'AnalyzedBlocks.txt'
        fileBlocks_FileName = os.path.abspath(self.cfg_params.get('CMSSW.fileblocks_file',defaultName))

        api = DBSAPI.dbsApi.DbsApi(args)
        self.files = self.queryDbs(api,path=self.datasetPath,runselection=runselection,useParent=useparent)

        # Check to see what the dataset is
        pdsName = self.datasetPath.split("/")[1]
        primDSs = api.listPrimaryDatasets(pdsName)
        dataType = primDSs[0]['Type']
        common.logger.debug("Datatype is %s" % dataType)
        if dataType == 'data' and not \
            (self.splitByRun or self.splitByLumi or self.splitDataByEvent):
            msg = 'Data must be split by lumi or by run. ' \
                  'Please see crab -help for the correct settings'
            raise  CrabException(msg)



        anFileBlocks = []
        if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)

        # parse files and fill arrays
        for file in self.files :
            parList  = []
            fileLumis = [] # List of tuples
            # skip already analyzed blocks
            fileblock = file['Block']['Name']
            if fileblock not in anFileBlocks :
                filename = file['LogicalFileName']
                # asked retry the list of parent for the given child
                if useparent==1:
                    parList = [x['LogicalFileName'] for x in file['ParentList']]
                if self.splitByLumi:
                    fileLumis = [ (x['RunNumber'], x['LumiSectionNumber'])
                                 for x in file['LumiList'] ]
                self.parent[filename] = parList
                # For LumiMask, intersection of two lists.
                if self.lumiMask and runselection:
                    self.lumis[filename] = runList.filterLumis(lumiList.filterLumis(fileLumis))
                elif runselection:
                    self.lumis[filename] = runList.filterLumis(fileLumis)
                elif self.lumiMask:
                    self.lumis[filename] = lumiList.filterLumis(fileLumis)
                else:
                    self.lumis[filename] = fileLumis
                if filename.find('.dat') < 0 :
                    events    = file['NumberOfEvents']
                    # Count number of events and lumis per block
                    if fileblock in self.eventsPerBlock.keys() :
                        self.eventsPerBlock[fileblock] += events
                    else :
                        self.eventsPerBlock[fileblock] = events
                    # Number of events per file
                    self.eventsPerFile[filename] = events
#.........这里部分代码省略.........

# 需要导入模块: from FWCore.PythonUtilities.LumiList import LumiList [as 别名]
# 或者: from FWCore.PythonUtilities.LumiList.LumiList import filterLumis [as 别名]

#.........这里部分代码省略.........
            #common.logger.info("looking up DBS2 ...")
            import DBSAPI.dbsApi
            import DBSAPI.dbsApiException
            start_time=time.time()
            api2 = DBSAPI.dbsApi.DbsApi(args)
            files2 = self.queryDbs(api2,path=self.datasetPath,runselection=runselection,useParent=useparent)
            elapsed=time.time() - start_time
            common.logger.info("DBS2 lookup took %5.2f sec" % elapsed)
            if useDBS2:
                self.files = files2
        if useDBS3 or verifyDBS23:
            #common.logger.info("looking up DBS3 ...")
            from dbs.apis.dbsClient import DbsApi
            start_time=time.time()
            api3 = DbsApi(dbs_url_3)
            files3 = self.queryDbs3(api3,path=self.datasetPath,runselection=runselection,useParent=useparent)
            elapsed=time.time() - start_time
            common.logger.info("DBS3 lookup took %5.2f sec" % elapsed)
            if useDBS3:
                self.files = files3
        if useDAS :
            self.files = self.queryDas(path=self.datasetPath,runselection=runselection,useParent=useparent)

        if verifyDBS23:
            if not self.compareFilesStructure(files2,files3):
                common.logger.info("ERROR: DBS2 - DB3 comparsion failed, please run crab -uploadLog and report to crabFeedback")
        

        # Check to see what the dataset is
        pdsName = self.datasetPath.split("/")[1]
        if useDBS2 :
            primDSs = api2.listPrimaryDatasets(pdsName)
            dataType = primDSs[0]['Type']
        elif useDBS3 :
            dataType=api3.listDataTypes(dataset=self.datasetPath)[0]['data_type']

        common.logger.info("Datatype is %s" % dataType)
        if dataType == 'data' and not \
            (self.splitByRun or self.splitByLumi or self.splitDataByEvent):
            msg = 'Data must be split by lumi or by run. ' \
                  'Please see crab -help for the correct settings'
            raise  CrabException(msg)



        anFileBlocks = []
        if self.skipBlocks: anFileBlocks = readTXTfile(self, fileBlocks_FileName)

        # parse files and fill arrays
        for file in self.files :
            parList  = []
            fileLumis = [] # List of tuples
            # skip already analyzed blocks
            fileblock = file['Block']['Name']
            if fileblock not in anFileBlocks :
                filename = file['LogicalFileName']
                # asked retry the list of parent for the given child
                if useparent==1:
                    parList = [x['LogicalFileName'] for x in file['ParentList']]
                if self.splitByLumi:
                    fileLumis = [ (x['RunNumber'], x['LumiSectionNumber'])
                                 for x in file['LumiList'] ]
                self.parent[filename] = parList
                # For LumiMask, intersection of two lists.
                if self.lumiMask and runselection:
                    self.lumis[filename] = runList.filterLumis(lumiList.filterLumis(fileLumis))
                elif runselection:
                    self.lumis[filename] = runList.filterLumis(fileLumis)
                elif self.lumiMask:
                    self.lumis[filename] = lumiList.filterLumis(fileLumis)
                else:
                    self.lumis[filename] = fileLumis

                if filename.find('.dat') < 0 :
                    events    = file['NumberOfEvents']
                    # Count number of events and lumis per block
                    if fileblock in self.eventsPerBlock.keys() :
                        self.eventsPerBlock[fileblock] += events
                    else :
                        self.eventsPerBlock[fileblock] = events
                    # Number of events per file
                    self.eventsPerFile[filename] = events

                    # List of files per block
                    if fileblock in self.blocksinfo.keys() :
                        self.blocksinfo[fileblock].append(filename)
                    else :
                        self.blocksinfo[fileblock] = [filename]

                    # total number of events
                    self.maxEvents += events
                    self.maxLumis  += len(self.lumis[filename])

        if  self.skipBlocks and len(self.eventsPerBlock.keys()) == 0:
            msg = "No new fileblocks available for dataset: "+str(self.datasetPath)
            raise  CrabException(msg)

        if len(self.eventsPerBlock) <= 0:
            msg="No data for %s in DBS\n Check datasetpath parameter in crab.cfg" % self.datasetPath
            raise  CrabException(msg)