Python ConfigSection.section_方法代码示例

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
    def testF(self):
        """
        Test internal functions pythonise_, listSections_
        """
        config = ConfigSection("config")

        config.section_("SectionA")
        config.section_("SectionB")
        config.SectionA.section_("Section1")
        config.SectionA.section_("Section2")
        config.SectionA.Section1.x   = 100
        config.SectionA.Section1.y   = 100

        pythonise = config.pythonise_()

        assert "config.section_('SectionA')"      in pythonise, "Pythonise failed: Could not find SectionA"
        assert "config.SectionA.Section1.x = 100" in pythonise, "Pythonise failed: Could not find x"

        pythonise = config.SectionA.pythonise_()

        assert "SectionA.section_('Section1')" in pythonise, "Pythonise failed: Could not find Section1"
        assert "SectionA.Section1.x = 100"     in pythonise, "Pythonise failed: Could not find x"

        self.assertEqual(config.listSections_(), ['SectionB', 'SectionA'])
        self.assertEqual(config.SectionA.listSections_(), ['Section2', 'Section1'])

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
    def reportWorkflowToDashboard(self, dashboardActivity):
        """
        _reportWorkflowToDashboard_
        Gathers workflow information from the arguments and reports it to the
        dashboard
        """
        try:
        #Create a fake config
            conf = ConfigSection()
            conf.section_('DashboardReporter')
            conf.DashboardReporter.dashboardHost = self.dashboardHost
            conf.DashboardReporter.dashboardPort = self.dashboardPort

            #Create the reporter
            reporter = DashboardReporter(conf)

            #Assemble the info
            workflow = {}
            workflow['name'] = self.workloadName
            workflow['application'] = self.frameworkVersion
            workflow['TaskType'] = dashboardActivity
            #Let's try to build information about the inputDataset
            dataset = 'DoesNotApply'
            if hasattr(self, 'inputDataset'):
                dataset = self.inputDataset
            workflow['datasetFull'] = dataset
            workflow['user'] = 'cmsdataops'

            #Send the workflow info
            reporter.addTask(workflow)
        except:
            #This is not critical, if it fails just leave it be
            logging.error("There was an error with dashboard reporting")

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
    def testH_ConfigSectionDictionariseInternalChildren(self):
        """
        The test checks if any item of the dictionary_whole_tree_()
        result is not unexpanded instance of ConfigSection.

        """
        config = ConfigSection("config")
        config.value1 = "MyValue1"
        config.section_("Task1")
        config.Task1.value2 = "MyValue2"
        config.Task1.section_("subSection")
        config.Task1.subSection.value3 = "MyValue3"
        d = config.dictionary_whole_tree_()
        for values in d.values():
            self.assertFalse(isinstance(values, ConfigSection))
        self.assertEqual(d["Task1"]["subSection"]["value3"], "MyValue3")

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
    def reportWorkflowToDashboard(self, dashboardActivity):
        """
        _reportWorkflowToDashboard_
        Gathers workflow information from the arguments and reports it to the
        dashboard
        """
        try:
            # Create a fake config
            conf = ConfigSection()
            conf.section_("DashboardReporter")
            conf.DashboardReporter.dashboardHost = self.dashboardHost
            conf.DashboardReporter.dashboardPort = self.dashboardPort

            # Create the reporter
            reporter = DashboardReporter(conf)

            # Assemble the info
            workflow = {}
            workflow["name"] = self.workloadName
            workflow["application"] = self.frameworkVersion
            workflow["scheduler"] = "BossAir"
            workflow["TaskType"] = dashboardActivity
            # Let's try to build information about the inputDataset
            dataset = "DoesNotApply"
            if hasattr(self, "inputDataset"):
                dataset = self.inputDataset
            workflow["datasetFull"] = dataset
            workflow["user"] = "cmsdataops"

            # These two make are not reported for now
            workflow["GridName"] = "NotAvailable"
            workflow["nevtJob"] = "NotAvailable"

            # Send the workflow info
            reporter.addTask(workflow)
        except:
            # This is not critical, if it fails just leave it be
            logging.error("There was an error with dashboard reporting")

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]

#.........这里部分代码省略.........
        """
        _persist_

        Pickle this object and save it to disk.
        """
        handle = open(filename, 'w')
        cPickle.dump(self.data, handle)
        handle.close()
        return

    def unpersist(self, filename, reportname = None):
        """
        _unpersist_

        Load a pickled FWJR from disk.
        """
        handle = open(filename, 'r')
        self.data = cPickle.load(handle)
        handle.close()

        # old self.report (if it existed) became unattached
        if reportname:
            self.report = getattr(self.data, reportname)

        return

    def addOutputModule(self, moduleName):
        """
        _addOutputModule_

        Add an entry for an output module.
        """
        self.report.outputModules.append(moduleName)
        self.report.output.section_(moduleName)

        outMod = getattr(self.report.output, moduleName)
        outMod.section_("files")
        outMod.section_("dataset")
        outMod.files.fileCount = 0

        return outMod

    def killOutput(self):
        """
        _killOutput_

        Remove all the output from the report.  This is useful for chained
        processing where we don't want to keep the output from a particular
        step in a job.
        """
        for outputModuleName in self.report.outputModules:
            delattr(self.report.output, outputModuleName)

        self.report.outputModules = []
        return

    def addOutputFile(self, outputModule, file = {}):
        """
        _addFile_

        Add an output file to the outputModule provided.
        """
        if not checkFileForCompletion(file):
            # Then the file is not complete, and should not be added
            print "ERROR"
            return None

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
"""
__MergeSample__

Example of a report from a merge job
Created on Fri Jun  8 13:22:30 2012

@author: dballest
"""

from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report

FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = '/Run195376-MuEG-Run2012B-PromptReco-v1-MuEG/DataProcessing/DataProcessingMergeSKIMStreamLogError'
FrameworkJobReport.workload = 'Unknown'
FrameworkJobReport.section_('cmsRun1')
FrameworkJobReport.cmsRun1.status = 0
FrameworkJobReport.cmsRun1.counter = 1
FrameworkJobReport.cmsRun1.section_('cleanup')
FrameworkJobReport.cmsRun1.cleanup.section_('unremoved')
FrameworkJobReport.cmsRun1.cleanup.section_('removed')
FrameworkJobReport.cmsRun1.cleanup.removed.fileCount = 0
FrameworkJobReport.cmsRun1.section_('errors')
FrameworkJobReport.cmsRun1.section_('logs')
FrameworkJobReport.cmsRun1.section_('parameters')
FrameworkJobReport.cmsRun1.parameters.GeneratorInfo = ''
FrameworkJobReport.cmsRun1.parameters.ReadBranches = ''
FrameworkJobReport.cmsRun1.outputModules = ['Merged']
FrameworkJobReport.cmsRun1.stopTime = 1338808530.44
FrameworkJobReport.cmsRun1.section_('site')
FrameworkJobReport.cmsRun1.section_('analysis')

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
Example of a report from a merge job
Created on Fri Jun  8 13:22:30 2012

@author: dballest
"""

from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report

FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = (
    "/Run195376-MuEG-Run2012B-PromptReco-v1-MuEG/DataProcessing/DataProcessingMergeSKIMStreamLogError"
)
FrameworkJobReport.workload = "Unknown"
FrameworkJobReport.section_("cmsRun1")
FrameworkJobReport.cmsRun1.status = 0
FrameworkJobReport.cmsRun1.counter = 1
FrameworkJobReport.cmsRun1.section_("cleanup")
FrameworkJobReport.cmsRun1.cleanup.section_("unremoved")
FrameworkJobReport.cmsRun1.cleanup.section_("removed")
FrameworkJobReport.cmsRun1.cleanup.removed.fileCount = 0
FrameworkJobReport.cmsRun1.section_("errors")
FrameworkJobReport.cmsRun1.section_("logs")
FrameworkJobReport.cmsRun1.section_("parameters")
FrameworkJobReport.cmsRun1.parameters.GeneratorInfo = ""
FrameworkJobReport.cmsRun1.parameters.ReadBranches = ""
FrameworkJobReport.cmsRun1.outputModules = ["Merged"]
FrameworkJobReport.cmsRun1.stopTime = 1338808530.44
FrameworkJobReport.cmsRun1.section_("site")
FrameworkJobReport.cmsRun1.section_("analysis")

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
class Aggregator(object):
    """
    _Aggregator_

    Util to aggregate performance reports for multicore jobs into a single
    performance report, including a multicore section to allow profiling of internal performance

    """
    def __init__(self):
        self.numCores = 0
        self.sections = {}
        self.values = {}
        self.report = ConfigSection("performance")
        #
        # populate the aggregator with the list of expected keys
        # based on the functions map above
        # create a combined performance report with the appropriate sections
        for red in AggrFunctions.keys():
            self.values[red] = []
            sect, param = red.split(".")
            if not self.sections.has_key(sect):
                self.sections[sect] = []
                self.report.section_(sect)
            if param not in self.sections[sect]:
                self.sections[sect].append(param)




    def add(self, perfRep):
        """
        _add_

        Add the contents of the given performance rep to this
        aggregator
        """
        self.numCores += 1
        for sect in self.sections.keys():
            for param in self.sections[sect]:
                key = ".".join([sect, param])
                try:
                    #protect against weird cases like NaNs in the reports
                    value = float(getSectParam(perfRep, sect, param))
                except AttributeError:
                    #protect against missing parameters
                    continue
                except ValueError:
                    continue
                self.values[key].append(value)


    def aggregate(self):
        """
        _aggregate_

        For each key in the map, run the appropriate aggregation function on it
        """
        for key, vals in self.values.items():
            # avoid divide by zero averages etc
            if len(vals) == 0: continue
            aggFunc = AggrFunctions[key]
            sect, param = key.split(".")
            section = getattr(self.report, sect)
            setattr(section, param, aggFunc(vals))
        self.createMulticoreSection()
        return self.report



    def createMulticoreSection(self):
        """
        _createMulticoreSection_

        create the multicore report section
        """
        self.report.section_("multicore")
        self.report.multicore.coresUsed = self.numCores
        if self.values.has_key("cpu.TotalJobTime"):
            vals = self.values["cpu.TotalJobTime"]
            self.report.multicore.summedProcessTime = sum(vals)
            self.report.multicore.averageProcessTime = average(vals)
            self.report.multicore.maxProcessTime = max(vals)
            self.report.multicore.minProcessTime = min(vals)
            self.report.multicore.processWaitingTime = max(vals) - min(vals)

            stepEffNom = float(sum(vals)) / float((max(vals) * self.numCores))
            stepEffDenom = float(average(vals))   / float(max(vals))
            stepEff = stepEffNom/stepEffDenom
            self.report.multicore.stepEfficiency = stepEff


        # frame in the merge report values
        # need to be set from the MulticoreCMSSW Executor
        self.report.multicore.mergeStartTime = None
        self.report.multicore.mergeEndTime = None
        self.report.multicore.numberOfMerges = None
        self.report.multicore.totalStepTime = None
        self.report.multicore.averageMergeTime = None
        self.report.multicore.maxMergeTime = None
        self.report.multicore.minMergeTime = None

# 需要导入模块: from WMCore.Configuration import ConfigSection [as 别名]
# 或者: from WMCore.Configuration.ConfigSection import section_ [as 别名]
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__FallbackSample__

Example of a report from a job that had xrootd fallback reads

"""

from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report

FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = '/Run195530-PhotonHad-Run2012B-PromptReco-v1-PhotonHad/DataProcessing'
FrameworkJobReport.workload = 'Unknown'
FrameworkJobReport.section_('cmsRun1')
FrameworkJobReport.cmsRun1.status = 1
FrameworkJobReport.cmsRun1.section_('cleanup')
FrameworkJobReport.cmsRun1.cleanup.section_('unremoved')
FrameworkJobReport.cmsRun1.cleanup.section_('removed')
FrameworkJobReport.cmsRun1.cleanup.removed.fileCount = 0
FrameworkJobReport.cmsRun1.section_('errors')
FrameworkJobReport.cmsRun1.section_('logs')
FrameworkJobReport.cmsRun1.section_('parameters')
FrameworkJobReport.cmsRun1.parameters.ReadBranches = ''
FrameworkJobReport.cmsRun1.outputModules = []
FrameworkJobReport.cmsRun1.section_('site')
FrameworkJobReport.cmsRun1.section_('analysis')
FrameworkJobReport.cmsRun1.analysis.section_('files')
FrameworkJobReport.cmsRun1.analysis.files.fileCount = 0
FrameworkJobReport.cmsRun1.section_('performance')