Python Configuration.ConfigSection类代码示例

    def testF(self):
        """
        Test internal functions pythonise_, listSections_
        """
        config = ConfigSection("config")

        config.section_("SectionA")
        config.section_("SectionB")
        config.SectionA.section_("Section1")
        config.SectionA.section_("Section2")
        config.SectionA.Section1.x   = 100
        config.SectionA.Section1.y   = 100

        pythonise = config.pythonise_()

        assert "config.section_('SectionA')"      in pythonise, "Pythonise failed: Could not find SectionA"
        assert "config.SectionA.Section1.x = 100" in pythonise, "Pythonise failed: Could not find x"

        pythonise = config.SectionA.pythonise_()

        assert "SectionA.section_('Section1')" in pythonise, "Pythonise failed: Could not find Section1"
        assert "SectionA.Section1.x = 100"     in pythonise, "Pythonise failed: Could not find x"

        self.assertEqual(config.listSections_(), ['SectionB', 'SectionA'])
        self.assertEqual(config.SectionA.listSections_(), ['Section2', 'Section1'])

    def reportWorkflowToDashboard(self, dashboardActivity):
        """
        _reportWorkflowToDashboard_
        Gathers workflow information from the arguments and reports it to the
        dashboard
        """
        try:
        #Create a fake config
            conf = ConfigSection()
            conf.section_('DashboardReporter')
            conf.DashboardReporter.dashboardHost = self.dashboardHost
            conf.DashboardReporter.dashboardPort = self.dashboardPort

            #Create the reporter
            reporter = DashboardReporter(conf)

            #Assemble the info
            workflow = {}
            workflow['name'] = self.workloadName
            workflow['application'] = self.frameworkVersion
            workflow['TaskType'] = dashboardActivity
            #Let's try to build information about the inputDataset
            dataset = 'DoesNotApply'
            if hasattr(self, 'inputDataset'):
                dataset = self.inputDataset
            workflow['datasetFull'] = dataset
            workflow['user'] = 'cmsdataops'

            #Send the workflow info
            reporter.addTask(workflow)
        except:
            #This is not critical, if it fails just leave it be
            logging.error("There was an error with dashboard reporting")

 def __init__(self, name):
     ConfigSection.__init__(self, name)
     self._internal_treetop = False
     self.section_("tree")
     self.tree.section_("children")
     self.tree.childNames = []
     self.tree.parent = None

 def testForwardSinkBasic(self):
     config = ConfigSection("forward")
     # address of the Processor, resp. Receiver to forward Alerts to
     config.address = self.address1
     config.controlAddr = self.controlAddr1
     config.label = "ForwardSinkTest"
     forwarder = ForwardSink(config)

def addTier1Skim(config, skimName, dataTier, primaryDataset, cmsswVersion,
                 processingVersion, configURL, globalTag, twoFileRead = False,
                 skimNode = None):
    """
    _addTier1Skim_

    Add the configuration of a skim that is to be run over a particular primary
    dataset and data tier at a particular site to the Tier0 configuration.  The
    skims will be launched as blocks are transfered to the site.  The site name
    must correspond to the site name in the ProdAgent JobQueue.
    """
    datasetConfig = config.Datasets.section_(primaryDataset)    

    skimConfig = ConfigSection(name = "SomeTier1Skim")
    skimConfig.PrimaryDataset = primaryDataset
    skimConfig.DataTier = dataTier
    skimConfig.SkimName = skimName
    skimConfig.CMSSWVersion = cmsswVersion
    skimConfig.ConfigURL = configURL
    skimConfig.GlobalTag = globalTag
    skimConfig.ProcessingVersion = processingVersion
    skimConfig.TwoFileRead = twoFileRead
    skimConfig.Node = skimNode

    datasetConfig.Tier1Skims.append(skimConfig)
    return

    def testH_ConfigSectionDictionariseInternalChildren(self):
        """
        The test checks if any item of the dictionary_whole_tree_()
        result is not unexpanded instance of ConfigSection.

        """
        config = ConfigSection("config")
        config.value1 = "MyValue1"
        config.section_("Task1")
        config.Task1.value2 = "MyValue2"
        config.Task1.section_("subSection")
        config.Task1.subSection.value3 = "MyValue3"
        d = config.dictionary_whole_tree_()
        for values in d.values():
            self.assertFalse(isinstance(values, ConfigSection))
        self.assertEqual(d["Task1"]["subSection"]["value3"], "MyValue3")

    def __init__(self, name="test"):
        ConfigSection.__init__(self, name)
        self.objectType = self.__class__.__name__
        #  //persistent data
        # //
        #//
        self.section_("persistency")
        self.persistency.specUrl = None
        #  //
        # // request related information
        #//
        self.section_("request")
        self.request.priority = None # what should be the default value
        #  //
        # // owner related information
        #//
        self.section_("owner")

        #  //
        # // Policies applied to this workload by the processing system
        #//
        self.section_("policies")
        self.policies.section_("start")
        self.policies.section_("end")
        self.policies.start.policyName = None
        self.policies.end.policyName = None

        #  //
        # // properties of the Workload and all tasks there-in
        #//
        self.section_("properties")
        self.properties.acquisitionEra = None
        self.properties.processingVersion = None
        self.properties.unmergedLFNBase = "/store/unmerged"
        self.properties.mergedLFNBase = "/store/data"
        self.properties.dashboardActivity = None

        #  //
        # // tasks
        #//
        self.section_("tasks")
        self.tasks.tasklist = []

        self.sandbox = None
        self.initialJobCount = 0

    def __init__(self, reportname = None):
        self.data = ConfigSection("FrameworkJobReport")
        self.data.steps = []
        self.data.workload = "Unknown"

        if reportname:
            self.addStep(reportname = reportname)

        return

    def reportWorkflowToDashboard(self, dashboardActivity):
        """
        _reportWorkflowToDashboard_
        Gathers workflow information from the arguments and reports it to the
        dashboard
        """
        try:
            # Create a fake config
            conf = ConfigSection()
            conf.section_("DashboardReporter")
            conf.DashboardReporter.dashboardHost = self.dashboardHost
            conf.DashboardReporter.dashboardPort = self.dashboardPort

            # Create the reporter
            reporter = DashboardReporter(conf)

            # Assemble the info
            workflow = {}
            workflow["name"] = self.workloadName
            workflow["application"] = self.frameworkVersion
            workflow["scheduler"] = "BossAir"
            workflow["TaskType"] = dashboardActivity
            # Let's try to build information about the inputDataset
            dataset = "DoesNotApply"
            if hasattr(self, "inputDataset"):
                dataset = self.inputDataset
            workflow["datasetFull"] = dataset
            workflow["user"] = "cmsdataops"

            # These two make are not reported for now
            workflow["GridName"] = "NotAvailable"
            workflow["nevtJob"] = "NotAvailable"

            # Send the workflow info
            reporter.addTask(workflow)
        except:
            # This is not critical, if it fails just leave it be
            logging.error("There was an error with dashboard reporting")

 def __init__(self):
     self.numCores = 0
     self.sections = {}
     self.values = {}
     self.report = ConfigSection("performance")
     #
     # populate the aggregator with the list of expected keys
     # based on the functions map above
     # create a combined performance report with the appropriate sections
     for red in AggrFunctions.keys():
         self.values[red] = []
         sect, param = red.split(".")
         if not self.sections.has_key(sect):
             self.sections[sect] = []
             self.report.section_(sect)
         if param not in self.sections[sect]:
             self.sections[sect].append(param)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__FallbackSample__

Example of a report from a job that had xrootd fallback reads

"""

from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report

FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = '/Run195530-PhotonHad-Run2012B-PromptReco-v1-PhotonHad/DataProcessing'
FrameworkJobReport.workload = 'Unknown'
FrameworkJobReport.section_('cmsRun1')
FrameworkJobReport.cmsRun1.status = 1
FrameworkJobReport.cmsRun1.section_('cleanup')
FrameworkJobReport.cmsRun1.cleanup.section_('unremoved')
FrameworkJobReport.cmsRun1.cleanup.section_('removed')
FrameworkJobReport.cmsRun1.cleanup.removed.fileCount = 0
FrameworkJobReport.cmsRun1.section_('errors')
FrameworkJobReport.cmsRun1.section_('logs')
FrameworkJobReport.cmsRun1.section_('parameters')
FrameworkJobReport.cmsRun1.parameters.ReadBranches = ''
FrameworkJobReport.cmsRun1.outputModules = []
FrameworkJobReport.cmsRun1.section_('site')
FrameworkJobReport.cmsRun1.section_('analysis')
FrameworkJobReport.cmsRun1.analysis.section_('files')
FrameworkJobReport.cmsRun1.analysis.files.fileCount = 0
FrameworkJobReport.cmsRun1.section_('performance')

 def getBaggage(self):
     confSect = ConfigSection()
     confSect.seeding = self.seeding
     confSect.lheInputFiles = self.lheInputFiles
     return confSect

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__MergeSample__

Example of a report from a merge job
Created on Fri Jun  8 13:22:30 2012

@author: dballest
"""

from WMCore.Configuration import ConfigSection
from WMCore.FwkJobReport.Report import Report

FrameworkJobReport = ConfigSection("FrameworkJobReport")
FrameworkJobReport.task = '/Run195376-MuEG-Run2012B-PromptReco-v1-MuEG/DataProcessing/DataProcessingMergeSKIMStreamLogError'
FrameworkJobReport.workload = 'Unknown'
FrameworkJobReport.section_('cmsRun1')
FrameworkJobReport.cmsRun1.status = 0
FrameworkJobReport.cmsRun1.counter = 1
FrameworkJobReport.cmsRun1.section_('cleanup')
FrameworkJobReport.cmsRun1.cleanup.section_('unremoved')
FrameworkJobReport.cmsRun1.cleanup.section_('removed')
FrameworkJobReport.cmsRun1.cleanup.removed.fileCount = 0
FrameworkJobReport.cmsRun1.section_('errors')
FrameworkJobReport.cmsRun1.section_('logs')
FrameworkJobReport.cmsRun1.section_('parameters')
FrameworkJobReport.cmsRun1.parameters.GeneratorInfo = ''
FrameworkJobReport.cmsRun1.parameters.ReadBranches = ''
FrameworkJobReport.cmsRun1.outputModules = ['Merged']
FrameworkJobReport.cmsRun1.stopTime = 1338808530.44

 def __init__(self, name = None):
     BaseConfigSection.__init__(self, name)

class Aggregator(object):
    """
    _Aggregator_

    Util to aggregate performance reports for multicore jobs into a single
    performance report, including a multicore section to allow profiling of internal performance

    """
    def __init__(self):
        self.numCores = 0
        self.sections = {}
        self.values = {}
        self.report = ConfigSection("performance")
        #
        # populate the aggregator with the list of expected keys
        # based on the functions map above
        # create a combined performance report with the appropriate sections
        for red in AggrFunctions.keys():
            self.values[red] = []
            sect, param = red.split(".")
            if not self.sections.has_key(sect):
                self.sections[sect] = []
                self.report.section_(sect)
            if param not in self.sections[sect]:
                self.sections[sect].append(param)




    def add(self, perfRep):
        """
        _add_

        Add the contents of the given performance rep to this
        aggregator
        """
        self.numCores += 1
        for sect in self.sections.keys():
            for param in self.sections[sect]:
                key = ".".join([sect, param])
                try:
                    #protect against weird cases like NaNs in the reports
                    value = float(getSectParam(perfRep, sect, param))
                except AttributeError:
                    #protect against missing parameters
                    continue
                except ValueError:
                    continue
                self.values[key].append(value)


    def aggregate(self):
        """
        _aggregate_

        For each key in the map, run the appropriate aggregation function on it
        """
        for key, vals in self.values.items():
            # avoid divide by zero averages etc
            if len(vals) == 0: continue
            aggFunc = AggrFunctions[key]
            sect, param = key.split(".")
            section = getattr(self.report, sect)
            setattr(section, param, aggFunc(vals))
        self.createMulticoreSection()
        return self.report



    def createMulticoreSection(self):
        """
        _createMulticoreSection_

        create the multicore report section
        """
        self.report.section_("multicore")
        self.report.multicore.coresUsed = self.numCores
        if self.values.has_key("cpu.TotalJobTime"):
            vals = self.values["cpu.TotalJobTime"]
            self.report.multicore.summedProcessTime = sum(vals)
            self.report.multicore.averageProcessTime = average(vals)
            self.report.multicore.maxProcessTime = max(vals)
            self.report.multicore.minProcessTime = min(vals)
            self.report.multicore.processWaitingTime = max(vals) - min(vals)

            stepEffNom = float(sum(vals)) / float((max(vals) * self.numCores))
            stepEffDenom = float(average(vals))   / float(max(vals))
            stepEff = stepEffNom/stepEffDenom
            self.report.multicore.stepEfficiency = stepEff


        # frame in the merge report values
        # need to be set from the MulticoreCMSSW Executor
        self.report.multicore.mergeStartTime = None
        self.report.multicore.mergeEndTime = None
        self.report.multicore.numberOfMerges = None
        self.report.multicore.totalStepTime = None
        self.report.multicore.averageMergeTime = None
        self.report.multicore.maxMergeTime = None
        self.report.multicore.minMergeTime = None