Python ConfigCache.getIDFromLabel方法代码示例

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import getIDFromLabel [as 别名]
 def _configCacheId(self, label):
     """Return config cache id for given config label"""
     key, cert = self.__class__.reqmgr['requests'].getKeyCert()
     configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert)
     try:
         configCacheId = configCache.getIDFromLabel(label)
     except:
         configCacheId = None
     if configCacheId:
         return configCacheId
     # The following will fail if FWCore.ParameterSet not in PYTHONPATH
     from PSetTweaks.WMTweak import makeTweak
     configCache.createUserGroup('test', 'test')
     configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs')
     configCache.addConfig(os.path.join(configDir, label + '.py'))
     configCache.setLabel(label)
     configCache.setDescription(label)
     modPath = imp.find_module(label, [configDir])
     loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
     configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
     configCache.save()
     return configCache.getIDFromLabel(label)

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import getIDFromLabel [as 别名]
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        self.injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                                   arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                                   arguments["CouchURL"], arguments["CouchDBName"])

        configCache = ConfigCache(arguments["CouchURL"], arguments["CouchDBName"])
        arguments["ProcConfigCacheID"] = configCache.getIDFromLabel(workloadName)
        
        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
        workload.setSiteWhitelist(arguments["CustodialSite"])
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import getIDFromLabel [as 别名]
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"]
        injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                              arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                              configCouchUrl, arguments["CouchDBName"],
                              arguments.get("EnvPath", None), arguments.get("BinPath", None))

        try:
            configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
            arguments["ConfigCacheID"] = configCache.getIDFromLabel(workloadName)
            if not arguments["ConfigCacheID"]:
                logging.error("The configuration was not uploaded to couch")
                raise Exception
        except Exception:
            logging.error("There was an exception loading the config out of the")
            logging.error("ConfigCache.  Check the scramOutput.log file in the")
            logging.error("PromptSkimScheduler directory to find out what went")
            logging.error("wrong.")
            raise

        parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
                                       'PromptSkim')
        arguments["ProcessingString"] = parsedProcVer["ProcString"]
        arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]

        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)

        # We need to strip off "MSS" as that causes all sorts of problems.
        if arguments["CustodialSite"].find("MSS") != -1:
            site = arguments["CustodialSite"][:-4]
        else:
            site = arguments["CustodialSite"]

        workload.setSiteWhitelist(site)
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload

# 需要导入模块: from WMCore.Cache.WMConfigCache import ConfigCache [as 别名]
# 或者: from WMCore.Cache.WMConfigCache.ConfigCache import getIDFromLabel [as 别名]

#.........这里部分代码省略.........
                                              doLogCollect = self.doLogCollect)
                recoMergeTasks[recoOutInfo['dataTier']] = mergeTask

            else:
                alcaTask = recoTask.addTask("AlcaSkim")

                scenarioArgs = { 'globalTag' : self.globalTag,
                                 'skims' : self.alcaSkims,
                                 'primaryDataset' : self.inputPrimaryDataset }
                if self.globalTagConnect:
                    scenarioArgs['globalTagConnect'] = self.globalTagConnect

                alcaOutMods = self.setupProcessingTask(alcaTask, taskType,
                                                       inputStep = recoTask.getStep("cmsRun1"),
                                                       inputModule = recoOutLabel,
                                                       scenarioName = self.procScenario,
                                                       scenarioFunc = "alcaSkim",
                                                       scenarioArgs = scenarioArgs,
                                                       splitAlgo = "WMBSMergeBySize",
                                                       splitArgs = {"max_merge_size": self.maxMergeSize,
                                                                    "min_merge_size": self.minMergeSize,
                                                                    "max_merge_events": self.maxMergeEvents},
                                                       stepType = cmsswStepType,
                                                       useMulticore = False)
                if self.doLogCollect:
                    self.addLogCollectTask(alcaTask, taskName = "AlcaSkimLogCollect")
                self.addCleanupTask(recoTask, recoOutLabel)

                for alcaOutLabel, alcaOutInfo in alcaOutMods.items():
                    self.addMergeTask(alcaTask, self.procJobSplitAlgo, alcaOutLabel,
                                      doLogCollect = self.doLogCollect)

        for promptSkim in self.promptSkims:
            if not promptSkim.DataTier in recoMergeTasks:
                error = 'PromptReco output does not have the following output data tier: %s.' % promptSkim.DataTier
                error += 'Please change the skim input to be one of the following: %s' % recoMergeTasks.keys()
                error += 'That should be in the relevant skimConfig in T0AST'
                logging.error(error)
                raise Exception

            mergeTask = recoMergeTasks[promptSkim.DataTier]
            skimTask = mergeTask.addTask(promptSkim.SkimName)
            parentCmsswStep = mergeTask.getStep('cmsRun1')

            parsedProcVer = parseT0ProcVer(promptSkim.ProcessingVersion,
                                           'PromptSkim')
            self.processingString = parsedProcVer["ProcString"]
            self.processingVersion = parsedProcVer["ProcVer"]

            if promptSkim.TwoFileRead:
                self.skimJobSplitArgs['include_parents'] = True
            else:
                self.skimJobSplitArgs['include_parents'] = False

            configLabel = '%s-%s' % (self.workloadName, promptSkim.SkimName)
            configCacheUrl = self.configCacheUrl or self.couchURL
            injectIntoConfigCache(self.frameworkVersion, self.scramArch,
                                       self.initCommand, promptSkim.ConfigURL, configLabel,
                                       configCacheUrl, self.couchDBName,
                                       self.envPath, self.binPath)
            try:
                configCache = ConfigCache(configCacheUrl, self.couchDBName)
                configCacheID = configCache.getIDFromLabel(configLabel)
                if configCacheID:
                    logging.error("The configuration was not uploaded to couch")
                    raise Exception
            except Exception:
                logging.error("There was an exception loading the config out of the")
                logging.error("ConfigCache.  Check the scramOutput.log file in the")
                logging.error("PromptSkimScheduler directory to find out what went")
                logging.error("wrong.")
                raise

            outputMods = self.setupProcessingTask(skimTask, "Skim", inputStep = parentCmsswStep, inputModule = "Merged",
                                                  couchURL = self.couchURL, couchDBName = self.couchDBName,
                                                  configCacheUrl = self.configCacheUrl,
                                                  configDoc = configCacheID, splitAlgo = self.skimJobSplitAlgo,
                                                  splitArgs = self.skimJobSplitArgs, useMulticore = False)
            if self.doLogCollect:
                self.addLogCollectTask(skimTask, taskName = "%sLogCollect" % promptSkim.SkimName)

            for outputModuleName in outputMods.keys():
                self.addMergeTask(skimTask, self.skimJobSplitAlgo, outputModuleName,
                                  doLogCollect = self.doLogCollect)

        workload.setBlockCloseSettings(self.blockCloseDelay,
                                       workload.getBlockCloseMaxFiles(),
                                       workload.getBlockCloseMaxEvents(),
                                       workload.getBlockCloseMaxSize())

        # setting the parameters which need to be set for all the tasks
        # sets acquisitionEra, processingVersion, processingString
        workload.setTaskPropertiesFromWorkload()

        # set the LFN bases (normally done by request manager)
        # also pass runNumber (workload evaluates it)
        workload.setLFNBase(self.mergedLFNBase, self.unmergedLFNBase,
                            runNumber = self.runNumber)

        return workload