Python StepFactory.getStepTemplate方法代码示例

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
 def testA(self):
     """template"""
     try:
         cmssw = StepFactory.getStepTemplate("CMSSW")
     except Exception, ex:
         msg = "Error loading Step Template of Type CMSSW\n"
         msg += str(ex)
         self.fail(msg)

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
 def testBuild(self):
     """ build a directory and verify it exists"""
     mytemplate = StepFactory.getStepTemplate("CMSSW")
     mystep = WMStep.makeWMStep("DummyStagingStep")
     mytemplate(mystep.data)
     self.testBuilder(mystep.data, "testTask", self.tempDir)
     self.assertTrue(os.path.exists(self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
    def setupNextSteps(self, task, origArgs):
        """
        _setupNextSteps_

        Modify the step one task to include N more CMSSW steps and
        chain the output between all three steps.
        """
        configCacheUrl = self.configCacheUrl or self.couchURL
        stepMapping = {}
        stepMapping.setdefault(origArgs['Step1']['StepName'], ('Step1', 'cmsRun1'))

        for i in range(2, self.stepChain + 1):
            currentStepNumber = "Step%d" % i
            currentCmsRun = "cmsRun%d" % i
            stepMapping.setdefault(origArgs[currentStepNumber]['StepName'], (currentStepNumber, currentCmsRun))
            taskConf = {}
            for k, v in origArgs[currentStepNumber].iteritems():
                taskConf[k] = v

            parentStepNumber = stepMapping.get(taskConf['InputStep'])[0]
            parentCmsRun = stepMapping.get(taskConf['InputStep'])[1]
            parentCmsswStep = task.getStep(parentCmsRun)
            parentCmsswStepHelper = parentCmsswStep.getTypeHelper()

            # Set default values for the task parameters
            self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
            globalTag = taskConf.get("GlobalTag", self.globalTag)
            frameworkVersion = taskConf.get("CMSSWVersion", self.frameworkVersion)
            scramArch = taskConf.get("ScramArch", self.scramArch)

            childCmssw = parentCmsswStep.addTopStep(currentCmsRun)
            childCmssw.setStepType("CMSSW")
            template = StepFactory.getStepTemplate("CMSSW")
            template(childCmssw.data)

            childCmsswStepHelper = childCmssw.getTypeHelper()
            childCmsswStepHelper.setGlobalTag(globalTag)
            childCmsswStepHelper.setupChainedProcessing(parentCmsRun, taskConf['InputFromOutputModule'])
            childCmsswStepHelper.cmsswSetup(frameworkVersion, softwareEnvironment="", scramArch=scramArch)
            childCmsswStepHelper.setConfigCache(configCacheUrl, taskConf['ConfigCacheID'], self.couchDBName)

            # Pileup check
            taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
            if taskConf["PileupConfig"]:
                self.setupPileup(task, taskConf['PileupConfig'])

            # Handling the output modules
            parentKeepOutput = strToBool(origArgs[parentStepNumber].get('KeepOutput', True))
            parentCmsswStepHelper.keepOutput(parentKeepOutput)
            childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
            childCmsswStepHelper.keepOutput(childKeepOutput)
            self.setupOutputModules(task, taskConf["ConfigCacheID"], currentCmsRun, childKeepOutput,
                                    taskConf['StepName'])

        # Closing out the task configuration. The last step output must be saved/merged
        childCmsswStepHelper.keepOutput(True)

        return

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
 def testCustomBuild(self):
     """ add in a custom directory and verify it gets created"""
     mytemplate = StepFactory.getStepTemplate("CMSSW")
     mystep = WMStep.makeWMStep("DummyStagingStep")
     mytemplate(mystep.data)
     helper = TemplateNS.Template.CoreHelper(mystep.data)
     helper.addDirectory("testdirectory1")
     helper.addDirectory("testdirectory2/testsubdir")
     self.testBuilder(mystep.data, "testTask", self.tempDir)
     self.assertTrue(os.path.exists(self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1" % self.tempDir))
     self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir" % (self.tempDir, "DummyStagingStep")))

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        newStepHelper = CMSSWStepHelper(newStep)
        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate(newStep)
        newStep.application.command.configuration = "PSet.py"
        return newStepHelper

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
    def buildWorkload(self):
        """
        _buildWorkload_

        Build the workload given all of the input parameters.

        Not that there will be LogCollect tasks created for each processing
        task and Cleanup tasks created for each merge task.

        """
        workload = self.createWorkload()
        workload.setDashboardActivity("tier0")
        self.reportWorkflowToDashboard(workload.getDashboardActivity())

        cmsswStepType = "CMSSW"
        taskType = "Processing"

        # complete output configuration
        for output in self.outputs:
            output['moduleLabel'] = "write_%s_%s" % (output['primaryDataset'],
                                                     output['dataTier'])

        # finalize splitting parameters
        mySplitArgs = self.expressSplitArgs.copy()
        mySplitArgs['algo_package'] = "T0.JobSplitting"

        expressTask = workload.newTask("Express")

        #
        # need to split this up into two separate code paths
        # one is direct reco from the streamer files
        # the other is conversion and then reco
        #
        if self.recoFrameworkVersion == None or self.recoFrameworkVersion == self.frameworkVersion:

            expressRecoStepName = "cmsRun1"

            scenarioArgs = { 'globalTag' : self.globalTag,
                             'globalTagTransaction' : self.globalTagTransaction,
                             'skims' : self.alcaSkims,
                             'dqmSeq' : self.dqmSequences,
                             'outputs' : self.outputs,
                             'inputSource' : "DAT" }

            if self.globalTagConnect:
                scenarioArgs['globalTagConnect'] = self.globalTagConnect

            expressOutMods = self.setupProcessingTask(expressTask, taskType,
                                                      scenarioName = self.procScenario,
                                                      scenarioFunc = "expressProcessing",
                                                      scenarioArgs = scenarioArgs,
                                                      splitAlgo = "Express",
                                                      splitArgs = mySplitArgs,
                                                      stepType = cmsswStepType,
                                                      forceUnmerged = True)
        else:

            expressRecoStepName = "cmsRun2"

            conversionOutMods = self.setupProcessingTask(expressTask, taskType,
                                                         scenarioName = self.procScenario,
                                                         scenarioFunc = "repack",
                                                         scenarioArgs = { 'outputs' : [ { 'dataTier' : "RAW",
                                                                                          'eventContent' : "ALL",
                                                                                          'primaryDataset' : self.specialDataset,
                                                                                          'moduleLabel' : "write_RAW" } ] },
                                                         splitAlgo = "Express",
                                                         splitArgs = mySplitArgs,
                                                         stepType = cmsswStepType,
                                                         forceUnmerged = True)

            # there is only one
            conversionOutLabel = conversionOutMods.keys()[0]

            # everything coming after should use the reco CMSSW version and Scram Arch
            self.frameworkVersion = self.recoFrameworkVersion
            self.scramArch = self.recoScramArch
            
            # add a second step doing the reconstruction
            parentCmsswStep = expressTask.getStep("cmsRun1")
            parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
            parentCmsswStepHelper.keepOutput(False)
            stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
            stepTwoCmssw.setStepType(cmsswStepType)

            template = StepFactory.getStepTemplate(cmsswStepType)
            template(stepTwoCmssw.data)

            stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()

            if self.multicore:
                # if multicore, poke in the number of cores setting
                stepTwoCmsswHelper.setNumberOfCores(self.multicoreNCores)

            stepTwoCmsswHelper.setGlobalTag(self.globalTag)
            stepTwoCmsswHelper.setupChainedProcessing("cmsRun1", conversionOutLabel)
            stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
                                          scramArch = self.scramArch)

            scenarioFunc = "expressProcessing"
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
    def setupNextSteps(self, task, origArgs):
        """
        _setupNextSteps_

        Modify the step one task to include N more CMSSW steps and
        chain the output between all three steps.
        """
        configCacheUrl = self.configCacheUrl or self.couchURL

        for i in range(2, self.stepChain + 1):
            inputStepName = "cmsRun%d" % (i-1)
            parentCmsswStep = task.getStep(inputStepName)
            parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
            parentCmsswStepHelper.keepOutput(False)

            currentStepName = "cmsRun%d" % i
            taskConf = {}
            for k, v in origArgs["Step%d" % i].iteritems():
                taskConf[k] = v
            # Set default values to task parameters
            self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
            globalTag = taskConf.get("GlobalTag", self.globalTag)

            childCmssw = parentCmsswStep.addTopStep(currentStepName)
            childCmssw.setStepType("CMSSW")
            template = StepFactory.getStepTemplate("CMSSW")
            template(childCmssw.data)

            childCmsswHelper = childCmssw.getTypeHelper()
            childCmsswHelper.setGlobalTag(globalTag)
            childCmsswHelper.setupChainedProcessing(inputStepName, taskConf['InputFromOutputModule'])
            # Assuming we cannot change the CMSSW version inside the same job
            childCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment="",
                                        scramArch=self.scramArch)
            childCmsswHelper.setConfigCache(configCacheUrl, taskConf['ConfigCacheID'],
                                            self.couchDBName)
            childCmsswHelper.keepOutput(False)

            # Pileup check
            taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
            if taskConf["PileupConfig"]:
                self.setupPileup(task, taskConf['PileupConfig'])

            # Handling the output modules
            outputMods = {}
            configOutput = self.determineOutputModules(configDoc=taskConf['ConfigCacheID'],
                                                       couchURL=configCacheUrl,
                                                       couchDBName=self.couchDBName)
            for outputModuleName in configOutput.keys():
                outputModule = self.addOutputModule(task, outputModuleName,
                                                    self.inputPrimaryDataset,
                                                    configOutput[outputModuleName]["dataTier"],
                                                    configOutput[outputModuleName]["filterName"],
                                                    stepName=currentStepName)
                outputMods[outputModuleName] = outputModule

        # Closing out the task configuration
        # Only the last step output is important :-)
        childCmsswHelper.keepOutput(True)
        self.addMergeTasks(task, currentStepName, outputMods)

        # Override task parameters by the workload ones in case of their absence
        self.updateCommonParams(task, taskConf)
        return

# 需要导入模块: from WMCore.WMSpec.Steps import StepFactory [as 别名]
# 或者: from WMCore.WMSpec.Steps.StepFactory import getStepTemplate [as 别名]
    def setupNextSteps(self, task, origArgs):
        """
        _setupNextSteps_

        Modify the step one task to include N more CMSSW steps and
        chain the output between all three steps.
        """
        self.stepParentageMapping.setdefault(origArgs['Step1']['StepName'], {})

        for i in range(2, self.stepChain + 1):
            currentStepNumber = "Step%d" % i
            currentCmsRun = "cmsRun%d" % i
            taskConf = {}
            for k, v in origArgs[currentStepNumber].items():
                taskConf[k] = v

            parentStepNumber = self.stepMapping.get(taskConf['InputStep'])[0]
            parentCmsRun = self.stepMapping.get(taskConf['InputStep'])[1]
            parentCmsswStep = task.getStep(parentCmsRun)
            parentCmsswStepHelper = parentCmsswStep.getTypeHelper()

            # Set default values for the task parameters
            self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
            globalTag = self.getStepValue('GlobalTag', taskConf, self.globalTag)
            frameworkVersion = self.getStepValue('CMSSWVersion', taskConf, self.frameworkVersion)
            scramArch = self.getStepValue('ScramArch', taskConf, self.scramArch)

            currentCmssw = parentCmsswStep.addTopStep(currentCmsRun)
            currentCmssw.setStepType("CMSSW")
            template = StepFactory.getStepTemplate("CMSSW")
            template(currentCmssw.data)

            currentCmsswStepHelper = currentCmssw.getTypeHelper()
            currentCmsswStepHelper.setGlobalTag(globalTag)
            currentCmsswStepHelper.setupChainedProcessing(parentCmsRun, taskConf['InputFromOutputModule'])
            currentCmsswStepHelper.cmsswSetup(frameworkVersion, softwareEnvironment="", scramArch=scramArch)
            currentCmsswStepHelper.setConfigCache(self.configCacheUrl, taskConf['ConfigCacheID'], self.couchDBName)

            # multicore settings
            multicore = self.multicore
            eventStreams = self.eventStreams
            if taskConf['Multicore'] > 0:
                multicore = taskConf['Multicore']
            if taskConf.get('EventStreams') >= 0:
                eventStreams = taskConf['EventStreams']

            currentCmsswStepHelper.setNumberOfCores(multicore, eventStreams)

            # Pileup check
            taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
            if taskConf["PileupConfig"]:
                self.setupPileup(task, taskConf['PileupConfig'], stepName=currentCmsRun)

            # Handling the output modules in order to decide whether we should
            # stage them out and report them in the Report.pkl file
            parentKeepOutput = strToBool(origArgs[parentStepNumber].get('KeepOutput', True))
            parentCmsswStepHelper.keepOutput(parentKeepOutput)
            childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
            currentCmsswStepHelper.keepOutput(childKeepOutput)
            self.setupOutputModules(task, taskConf, currentCmsRun, childKeepOutput)

        # Closing out the task configuration. The last step output must be saved/merged
        currentCmsswStepHelper.keepOutput(True)

        return