Python Configuration.Configuration类代码示例

    def __init__(self, **configDict):
        """
        init jobCreator
        """

        myThread = threading.currentThread()

        self.transaction = myThread.transaction

        # DAO factory for WMBS objects
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi)

        # WMCore splitter factory for splitting up jobs.
        self.splitterFactory = SplitterFactory()

        config = Configuration()
        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl = configDict["couchURL"]
        config.JobStateMachine.couch_retries = configDict["defaultRetries"]
        config.JobStateMachine.couchDBName = configDict["couchDBName"]

        self.config = config

        # Variables
        self.jobCacheDir = configDict["jobCacheDir"]
        self.defaultJobType = configDict["defaultJobType"]
        self.limit = configDict.get("fileLoadLimit", 500)

        self.createWorkArea = CreateWorkArea()

        self.changeState = ChangeState(self.config)

        return

    def testD(self):
        """test documentation"""


        config = Configuration()
        config.section_("Section1")
        config.Section1.Parameter1 = True
        config.Section1.Parameter2 = "string"
        config.Section1.Parameter3 = 123
        config.Section1.Parameter4 = 123.456
        config.Section1.Parameter5 = {
            "test1" : "test2", "test3" : 123
            }


        config.Section1.document_("""This is Section1""")
        config.Section1.document_("""This is Section1.Parameter1""",
                                  "Parameter1")
        config.Section1.document_("""This is Section1.Parameter2""",
                                  "Parameter2")
        config.Section1.document_("""This is Section1.Parameter3\n with multiline comments""",
                                  "Parameter3")


        try:
            config.Section1.documentedString_()
        except Exception, ex:
            msg = "Error calling ConfigSection.documentedString_:\n"
            msg += "%s\n" % str(ex)
            self.fail(msg)

    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("RetryManager")
        config.RetryManager.logLevel     = 'DEBUG'
        config.RetryManager.namespace    = 'WMComponent.RetryManager.RetryManager'
        config.RetryManager.maxRetries   = 10
        config.RetryManager.pollInterval = 10
        # These are the cooloff times for the RetryManager, the times it waits
        # Before attempting resubmission
        config.RetryManager.coolOffTime  = {'create': 120, 'submit': 120, 'job': 120}
        # Path to plugin directory
        config.RetryManager.pluginPath   = 'WMComponent.RetryManager.PlugIns'
        #config.RetryManager.pluginName   = ''
        config.RetryManager.WMCoreBase   = WMCore.WMInit.getWMBASE()
        config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components')


        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = "retry_manager_t"

        return config

    def getConfig(self, dbs3UploadOnly = False):
        """
        _getConfig_

        This creates the actual config file used by the component.
        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat  = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("DBS3Upload")
        config.DBS3Upload.pollInterval     = 10
        config.DBS3Upload.logLevel         = 'DEBUG'
        #config.DBS3Upload.dbsUrl           = "https://cmsweb-testbed.cern.ch/dbs/dev/global/DBSWriter"
        #config.DBS3Upload.dbsUrl           = "https://dbs3-dev01.cern.ch/dbs/prod/global/DBSWriter"
        config.DBS3Upload.dbsUrl           = self.dbsUrl
        config.DBS3Upload.namespace        = 'WMComponent.DBS3Buffer.DBSUpload'
        config.DBS3Upload.componentDir     = os.path.join(os.getcwd(), 'Components')
        config.DBS3Upload.nProcesses       = 1
        config.DBS3Upload.dbsWaitTime      = 0.1
        config.DBS3Upload.datasetType      = "VALID"
        config.DBS3Upload.dbs3UploadOnly   = dbs3UploadOnly
        return config

    def testC(self):
        """add components"""

        config = Configuration()
        config.component_("Component1")
        config.component_("Component2")
        config.component_("Component3")

        comp1 = getattr(config, "Component1", None)
        self.failUnless(comp1 != None)
        comp2 = getattr(config, "Component2", None)
        self.failUnless(comp2 != None)

    def getConfig(self):
        """
        _getConfig_

        """
        config = Configuration()

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = 'wmagent_jobdump'

        return config

    def createConfig(self):
        """
        Create a config object for testing

        """

        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.section_("DBSInterface")
        config.DBSInterface.globalDBSUrl     = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
        config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
        config.DBSInterface.DBSUrl           = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
        config.DBSInterface.DBSVersion       = 'DBS_2_0_9'
        config.DBSInterface.DBSBlockMaxFiles = 10
        config.DBSInterface.DBSBlockMaxSize  = 9999999999
        config.DBSInterface.DBSBlockMaxTime  = 10000
        config.DBSInterface.MaxFilesToCommit = 10

        return config

    def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
        """ 
        Loads (if available) your configuration file and augments
        it with the standard settings used in multiple tests.
        """
        if configurationFile != None:
            config = loadConfigurationFile(configurationFile)
        else:
            config = Configuration()

        # some general settings that would come from the general default
        # config file
        config.Agent.contact = "[email protected]"
        config.Agent.teamName = "Lakers"
        config.Agent.agentName = "Lebron James"

        config.section_("General")
        # If you need a testDir, call testInit.generateWorkDir
        # config.General.workDir = os.getenv("TESTDIR")

        config.section_("CoreDatabase")
        if connectUrl:
            config.CoreDatabase.connectUrl = connectUrl
            config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
            config.CoreDatabase.socket = socket or os.getenv("DBSOCK") 
        else:
            if (os.getenv('DATABASE') == None):
                raise RuntimeError, \
                    "You must set the DATABASE environment variable to run tests"
            config.CoreDatabase.connectUrl = os.getenv("DATABASE")
            config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
            config.CoreDatabase.socket = os.getenv("DBSOCK")
            if os.getenv("DBHOST"):
                print "****WARNING: the DBHOST environment variable will be deprecated soon***"
                print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
            # after this you can augment it with whatever you need.
        return config

    def testB(self):
        """add settings"""


        config = Configuration()
        config.section_("Section1")

        section1 = getattr(config, "Section1", None)
        self.failUnless(section1 != None)

        config.section_("Section2")
        section2 = getattr(config, "Section2", None)
        self.failUnless(section2 != None)

        self.assertRaises(AttributeError, getattr, config, "Section3")

        # basic types
        config.Section1.Parameter1 = True
        config.Section1.Parameter2 = "string"
        config.Section1.Parameter3 = 123
        config.Section1.Parameter4 = 123.456




        self.assertEqual(config.Section1.Parameter1, True)
        self.assertEqual(config.Section1.Parameter2, "string")
        self.assertEqual(config.Section1.Parameter3, 123)
        self.assertEqual(config.Section1.Parameter4, 123.456)

        # dictionary format:
        try:
            section1Dict = config.Section1.dictionary_()
        except Exception, ex:
            msg = "Error converting section to dictionary:\n"
            msg += "%s\n" % str(ex)
            self.fail(msg)

    def testE(self):
        """test save/load """

        testValues = [
            "string", 123, 123.456,
            ["list", 789, 10.1 ],
            { "dict1" : "value", "dict2" : 10.0 }
            ]

        config = Configuration()
        for x in range(0, 5):
            config.section_("Section%s" % x)
            config.component_("Component%s" % x)
            sect = getattr(config, "Section%s" % x)
            comp = getattr(config, "Component%s" % x)
            sect.document_("This is Section%s" % x)
            comp.document_("This is Component%s" % x)

            for i in range(0, 5):
                setattr(comp, "Parameter%s" % i, testValues[i])
                setattr(sect, "Parameter%s" % i, testValues[i])
                comp.document_("This is Parameter%s" % i,
                               "Parameter%s" %i)
                sect.document_("This is Parameter%s" %i,
                               "Parameter%s" %i)

        stringSave = str(config)
        documentSave = config.documentedString_()
        commentSave = config.commentedString_()


        saveConfigurationFile(config, self.normalSave)
        saveConfigurationFile(config, self.docSave, document = True)
        saveConfigurationFile(config, self.commentSave, comment = True)

        plainConfig = loadConfigurationFile(self.normalSave)

        docConfig = loadConfigurationFile(self.docSave)


        commentConfig = loadConfigurationFile(self.commentSave)

    def createConfig(self):
        """
        _createConfig_

        Create a config and save it to the temp dir.  Set the WMAGENT_CONFIG
        environment variable so the config gets picked up.
        """
        config = Configuration()
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())
        config.section_("Agent")
        config.Agent.componentName = "resource_control_t"
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
        configHandle.write(str(config))
        configHandle.close()

        os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
        return

    def getConfig(self):
        """
        _getConfig_

        Gets a basic config from default location
        """

        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName       = 'testAgent'
        config.Agent.componentName   = self.componentName
        config.Agent.useHeartbeat    = False


        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        #Now the CoreDatabase information
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # BossAir and MockPlugin configuration
        config.section_("BossAir")
        config.BossAir.pluginNames = ['MockPlugin']
        config.BossAir.pluginDir   = 'WMCore.BossAir.Plugins'
        config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
        config.BossAir.nCondorProcesses = 1
        config.BossAir.section_("MockPlugin")
        config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
                                                         'WMComponent_t/JobSubmitter_t',
                                                         "submit.sh")
        # JobSubmitter configuration
        config.component_("JobSubmitter")
        config.JobSubmitter.logLevel      = 'DEBUG'
        config.JobSubmitter.maxThreads    = 1
        config.JobSubmitter.pollInterval  = 10
        config.JobSubmitter.submitScript  = os.path.join(getTestBase(),
                                                         'WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(self.testDir, 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200

        #JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName     = "jobsubmitter_t"
        config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'

        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.componentDir)

        return config

    def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
        """
        Loads (if available) your configuration file and augments
        it with the standard settings used in multiple tests.
        """
        if configurationFile != None:
            config = loadConfigurationFile(configurationFile)
        else:
            config = Configuration()

        # some general settings that would come from the general default
        # config file
        config.Agent.contact = "[email protected]"
        config.Agent.teamName = "Lakers"
        config.Agent.agentName = "Lebron James"
        config.Agent.hostName = "testhost.laker.world"

        config.section_("General")
        # If you need a testDir, call testInit.generateWorkDir
        # config.General.workDir = os.getenv("TESTDIR")

        config.section_("CoreDatabase")
        if connectUrl:
            config.CoreDatabase.connectUrl = connectUrl
            config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
            config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
        else:
            if os.getenv('DATABASE') == None:
                raise RuntimeError("You must set the DATABASE environment variable to run tests")
            config.CoreDatabase.connectUrl = os.getenv("DATABASE")
            config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
            config.CoreDatabase.socket = os.getenv("DBSOCK")
            if os.getenv("DBHOST"):
                print("****WARNING: the DBHOST environment variable will be deprecated soon***")
                print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
            # after this you can augment it with whatever you need.

        couchurl = os.getenv("COUCHURL")
        config.section_("ACDC")
        config.ACDC.couchurl = couchurl
        config.ACDC.database = "wmagent_acdc_t"

        config.component_("JobStateMachine")
        config.JobStateMachine.couchurl = couchurl
        config.JobStateMachine.couchDBName = "wmagent_job_test"
        config.JobStateMachine.jobSummaryDBName = "job_summary"
        config.JobStateMachine.summaryStatsDBName = "stat_summary_test"

        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.componentDir = os.getcwd()
        config.JobAccountant.logLevel = 'SQLDEBUG'

        config.component_("TaskArchiver")
        config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
        config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
        config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"

        return config

#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for DBSBuffer specific
parameters.
"""
__all__ = []




from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("DBSBuffer")
#The log level of the component. 
config.DBSBuffer.logLevel = 'INFO'
#The namespace of the buffer. Used to load the module as daemon
config.DBSBuffer.namespace = 'WMComponent.DBSBuffer.DBSBuffer'
# maximum number of threads we want to deal
# with messages per pool.
config.DBSBuffer.maxThreads = 1
#
# JobSuccess Handler
#
config.DBSBuffer.jobSuccessHandler = \
    'WMComponent.DBSBuffer.Handler.JobSuccess'

#!/usr/bin/env python
#pylint: disable-msg=E1101,E1103,C0103,R0902
"""
Defines default config values for CranWmbs specific
parameters.
"""
__all__ = []
__revision__ = "$Id: DefaultConfig.py,v 0.2 2009/09/30 01:26:34 hriahi Exp $"
__version__ = "$Revision: 0.2 $"

import os
from WMCore.Agent.Configuration import Configuration

config = Configuration()
config.component_("CrabJobCreator")
config.CrabJobCreator.logLevel = "DEBUG"
config.CrabJobCreator.componentName = "CrabJobCreator"
config.CrabJobCreator.componentDir = \
    os.path.join(os.getenv("TESTDIR"), "CrabJobCreator")

# The maximum number of threads to process each message type
config.CrabJobCreator.maxThreads = 10

# CrabServer parameter
config.CrabJobCreator.wdir = '/data/Storage/logs' 
config.CrabJobCreator.maxRetries = 3
config.CrabJobCreator.credentialType = "Proxy"
config.CrabJobCreator.ProxiesDir = "/tmp/del_proxies/"
config.CrabJobCreator.StorageName = "crab.pg.infn.it"
config.CrabJobCreator.storagePort = "2811"
config.CrabJobCreator.Protocol = "gridftp"