Python Steps.StepFactory类代码示例

    def setUp(self):
        # stolen from CMSSWExecutor_t. thanks, dave
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")
        stepHelper = step = self.task.makeStep("DeleteTest")
        self.step = stepHelper.data
        self.actualStep = stepHelper
        template = DeleteTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())

        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.sandboxDir = "%s/UnitTests" % self.testDir

        self.task.build(self.testDir)
        sys.path.insert(0, self.testDir)
        sys.path.insert(0, self.sandboxDir)


        self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")

        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)

    def testB_ExecuteNonZeroExit(self):
        """
        _ExecuteNonZeroExit_

        Test the execution of a script
        which exits with non-zero code.
        """
        self.step.application.command.executable = "brokenCmsRun.py"
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            try:
                executor.execute()
                self.fail("An exception should have been raised")
            except WMExecutionFailure as ex:
                executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
                self.assertEqual(8001, executor.report.getExitCode())
                report = Report()
                report.load("Report.pkl")
                self.assertEqual(8001, report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

    def testC_ExecuteSegfault(self):
        """
        _ExecuteSegfault_

        Test the execution of a script
        which raises a ABRT signal which
        is the normal CMSSW response
        to a SEGFAULT.
        """
        self.step.application.command.executable = "test.sh"
        # CMSSW leaves an empty FWJR when a SEGFAULT is present
        open(os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"), "w").close()
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            try:
                executor.execute()
                self.fail("An exception should have been raised")
            except WMExecutionFailure as ex:
                executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
                self.assertEqual(50115, executor.report.getExitCode())
                report = Report()
                report.load("Report.pkl")
                self.assertEqual(50115, report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

    def testD_ExecuteNoOutput(self):
        """
        _ExecuteNoOutput_

        Test what happens when no output is produced,
        the proper error should be included.
        """
        self.step.application.command.executable = "cmsRun.py"
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            executor.execute()
            executor.post()
            self.assertEqual(60450, executor.report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

 def testC(self):
     """builder"""
     try:
         cmssw = StepFactory.getStepBuilder("CMSSW")
     except Exception as ex:
         msg = "Error loading Step Builder of Type CMSSW\n"
         msg += str(ex)
         self.fail(msg)

 def testA(self):
     """template"""
     try:
         cmssw = StepFactory.getStepTemplate("CMSSW")
     except Exception, ex:
         msg = "Error loading Step Template of Type CMSSW\n"
         msg += str(ex)
         self.fail(msg)

 def testC(self):
     """emulator"""
     try:
         cmssw = StepFactory.getStepEmulator("CMSSW")
     except Exception, ex:
         msg = "Error loading Step Emulator of Type CMSSW\n"
         msg += str(ex)
         self.fail(msg)

 def testBuild(self):
     """ build a directory and verify it exists"""
     mytemplate = StepFactory.getStepTemplate("CMSSW")
     mystep = WMStep.makeWMStep("DummyStagingStep")
     mytemplate(mystep.data)
     self.testBuilder(mystep.data, "testTask", self.tempDir)
     self.assertTrue(os.path.exists(self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))

    def setupNextSteps(self, task, origArgs):
        """
        _setupNextSteps_

        Modify the step one task to include N more CMSSW steps and
        chain the output between all three steps.
        """
        configCacheUrl = self.configCacheUrl or self.couchURL
        stepMapping = {}
        stepMapping.setdefault(origArgs['Step1']['StepName'], ('Step1', 'cmsRun1'))

        for i in range(2, self.stepChain + 1):
            currentStepNumber = "Step%d" % i
            currentCmsRun = "cmsRun%d" % i
            stepMapping.setdefault(origArgs[currentStepNumber]['StepName'], (currentStepNumber, currentCmsRun))
            taskConf = {}
            for k, v in origArgs[currentStepNumber].iteritems():
                taskConf[k] = v

            parentStepNumber = stepMapping.get(taskConf['InputStep'])[0]
            parentCmsRun = stepMapping.get(taskConf['InputStep'])[1]
            parentCmsswStep = task.getStep(parentCmsRun)
            parentCmsswStepHelper = parentCmsswStep.getTypeHelper()

            # Set default values for the task parameters
            self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
            globalTag = taskConf.get("GlobalTag", self.globalTag)
            frameworkVersion = taskConf.get("CMSSWVersion", self.frameworkVersion)
            scramArch = taskConf.get("ScramArch", self.scramArch)

            childCmssw = parentCmsswStep.addTopStep(currentCmsRun)
            childCmssw.setStepType("CMSSW")
            template = StepFactory.getStepTemplate("CMSSW")
            template(childCmssw.data)

            childCmsswStepHelper = childCmssw.getTypeHelper()
            childCmsswStepHelper.setGlobalTag(globalTag)
            childCmsswStepHelper.setupChainedProcessing(parentCmsRun, taskConf['InputFromOutputModule'])
            childCmsswStepHelper.cmsswSetup(frameworkVersion, softwareEnvironment="", scramArch=scramArch)
            childCmsswStepHelper.setConfigCache(configCacheUrl, taskConf['ConfigCacheID'], self.couchDBName)

            # Pileup check
            taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
            if taskConf["PileupConfig"]:
                self.setupPileup(task, taskConf['PileupConfig'])

            # Handling the output modules
            parentKeepOutput = strToBool(origArgs[parentStepNumber].get('KeepOutput', True))
            parentCmsswStepHelper.keepOutput(parentKeepOutput)
            childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
            childCmsswStepHelper.keepOutput(childKeepOutput)
            self.setupOutputModules(task, taskConf["ConfigCacheID"], currentCmsRun, childKeepOutput,
                                    taskConf['StepName'])

        # Closing out the task configuration. The last step output must be saved/merged
        childCmsswStepHelper.keepOutput(True)

        return

 def setUp(self):
     '''create the builder object and the temporary directory'''
     self.tempDir = tempfile.mkdtemp()
     
     try:
         self.testBuilder = StepFactory.getStepBuilder("StageOut")
     except Exception, ex:
         msg = "Failed to instantiate Builder:\n"
         msg += str(ex)
         self.fail(msg)      

    def setUp(self):
        """create the builder object and the temporary directory"""
        self.tempDir = tempfile.mkdtemp()

        try:
            self.testBuilder = StepFactory.getStepBuilder("CMSSW")
        except Exception as ex:
            msg = "Failed to instantiate Builder:\n"
            msg += str(ex)
            self.fail(msg)

    def addEmulator(self, nodeName, emulatorName):
        """
        _addEmulator_

        Add an Emulator for the node provided, emulatorName is the name
        of the emulator class to be loaded by the Emulator factory

        TODO: Exception handling

        """
        emuInstance = StepFactory.getStepEmulator(emulatorName)
        self.emulators[nodeName] = emuInstance
        return

 def testCustomBuild(self):
     """ add in a custom directory and verify it gets created"""
     mytemplate = StepFactory.getStepTemplate("CMSSW")
     mystep = WMStep.makeWMStep("DummyStagingStep")
     mytemplate(mystep.data)
     helper = TemplateNS.Template.CoreHelper(mystep.data)
     helper.addDirectory("testdirectory1")
     helper.addDirectory("testdirectory2/testsubdir")
     self.testBuilder(mystep.data, "testTask", self.tempDir)
     self.assertTrue(os.path.exists(self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
     self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1" % self.tempDir))
     self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir" % (self.tempDir, "DummyStagingStep")))

    def __call__(self, task):
        """
        _operator(task)_

        Invoke the builder on the task provided

        TODO: Exception handling

        """
        taskName = task.getPathName()
        self.taskSpace = initialiseWMTaskSpace(self.workDir, taskName)
        for step in task.steps().nodeIterator():
            stepType = step.stepType
            builder = StepFactory.getStepBuilder(stepType)
            builder(step, task.name(), self.taskSpace)

    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        newStepHelper = CMSSWStepHelper(newStep)
        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate(newStep)
        newStep.application.command.configuration = "PSet.py"
        return newStepHelper