Python HeartbeatAPI.registerComponent方法代码示例

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
    def testHeartbeat(self):
        testComponent = HeartbeatAPI("testComponent")
        testComponent.pollInterval = 10
        testComponent.registerComponent()
        self.assertEqual(testComponent.getHeartbeatInfo(), [])

        testComponent.updateWorkerHeartbeat("testWorker")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['worker_name'], "testWorker")
        time.sleep(1)

        testComponent.updateWorkerHeartbeat("testWorker2")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['worker_name'], "testWorker2")

        time.sleep(1)
        testComponent.updateWorkerHeartbeat("testWorker")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['worker_name'], "testWorker")


        testComponent = HeartbeatAPI("test2Component")
        testComponent.pollInterval = 20
        testComponent.registerComponent()
        time.sleep(1)
        testComponent.updateWorkerHeartbeat("test2Worker")

        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 2)
        self.assertEqual(result[0]['worker_name'], "testWorker")
        self.assertEqual(result[1]['worker_name'], "test2Worker")

        time.sleep(1)
        testComponent.updateWorkerHeartbeat("test2Worker2")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 2)
        self.assertEqual(result[0]['worker_name'], "testWorker")
        self.assertEqual(result[1]['worker_name'], "test2Worker2")

        time.sleep(1)
        testComponent.updateWorkerHeartbeat("test2Worker")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(len(result), 2)
        self.assertEqual(result[0]['worker_name'], "testWorker")
        self.assertEqual(result[1]['worker_name'], "test2Worker")

        testComponent.updateWorkerError("test2Worker", "Error1")
        result = testComponent.getHeartbeatInfo()
        self.assertEqual(result[1]['error_message'], "Error1")

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
    def setUp(self):
        """
        _setUp_

        Set up vital components
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
                                                 'WMCore.ResourceControl', 'WMCore.ThreadPool',
                                                 'WMCore.Agent.Database'],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)



        locationAction = self.daoFactory(classname = "Locations.New")
        pendingSlots  = self.daoFactory(classname = "Locations.SetPendingSlots")


        for site in self.sites:
            locationAction.execute(siteName = site, seName = 'se.%s' % (site), ceName = site)
            pendingSlots.execute(siteName = site, pendingSlots = 1000)


        #Create sites in resourceControl
        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site)
            resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
                                            maxSlots = 10000, pendingSlots = 10000)


        self.testDir = self.testInit.generateWorkDir()


        # Set heartbeat
        for component in self.components:
            heartbeatAPI = HeartbeatAPI(component)
            heartbeatAPI.registerComponent()




        return

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
    def testAddComponent(self):
        """
        _testAddComponent_

        Test creation of components and worker threads as well as the
        get heartbeat DAOs
        """
        comp1 = HeartbeatAPI("testComponent1", pollInterval=60, heartbeatTimeout=600)
        comp1.registerComponent()
        self.assertEqual(comp1.getHeartbeatInfo(), [])  # no worker thread yet

        comp1.registerWorker("testWorker1")
        self.assertEqual(len(comp1.getHeartbeatInfo()), 1)

        comp1.registerWorker("testWorker2")
        self.assertEqual(len(comp1.getHeartbeatInfo()), 2)

        comp2 = HeartbeatAPI("testComponent2", pollInterval=30, heartbeatTimeout=300)
        comp2.registerComponent()
        self.assertEqual(comp2.getHeartbeatInfo(), [])  # no worker thread yet
        self.assertEqual(len(comp2.getAllHeartbeatInfo()), 2)

        comp2.registerWorker("testWorker21")
        self.assertEqual(len(comp2.getHeartbeatInfo()), 1)
        self.assertEqual(len(comp2.getAllHeartbeatInfo()), 3)

        comp1.updateWorkerHeartbeat("testWorker1", "Running")
        comp1.updateWorkerHeartbeat("testWorker2", "Running")
        comp2.updateWorkerHeartbeat("testWorker21", "Running")
        self.assertEqual(len(comp1.getAllHeartbeatInfo()), 3)
        self.assertEqual(len(comp2.getAllHeartbeatInfo()), 3)

        comp1Res = comp1.getHeartbeatInfo()
        comp2Res = comp2.getHeartbeatInfo()
        self.assertEqual(len(comp1Res), 2)
        self.assertEqual(len(comp2Res), 1)

        self.assertItemsEqual([item["name"] for item in comp1Res], ["testComponent1", "testComponent1"])
        self.assertItemsEqual([item["worker_name"] for item in comp1Res], ["testWorker1", "testWorker2"])
        self.assertItemsEqual([item["state"] for item in comp1Res], ["Running", "Running"])
        self.assertItemsEqual([item["poll_interval"] for item in comp1Res], [60, 60])
        self.assertItemsEqual([item["update_threshold"] for item in comp1Res], [600, 600])

        self.assertItemsEqual([item["name"] for item in comp2Res], ["testComponent2"])
        self.assertItemsEqual([item["worker_name"] for item in comp2Res], ["testWorker21"])
        self.assertItemsEqual([item["state"] for item in comp2Res], ["Running"])
        self.assertItemsEqual([item["poll_interval"] for item in comp2Res], [30])
        self.assertItemsEqual([item["update_threshold"] for item in comp2Res], [300])

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
    def testUpdateWorkers(self):
        """
        _testUpdateWorkers_

        Create a couple of components and workers and test the update methods
        """
        comp1 = HeartbeatAPI("testComponent1", pollInterval=60, heartbeatTimeout=600)
        comp1.registerComponent()
        comp1.registerWorker("testWorker1")
        comp1.registerWorker("testWorker2")

        comp2 = HeartbeatAPI("testComponent2", pollInterval=30, heartbeatTimeout=300)
        comp2.registerComponent()
        comp2.registerWorker("testWorker21")

        comp1.updateWorkerCycle("testWorker1", 1.001, None)
        comp2.updateWorkerCycle("testWorker21", 1234.1, 100)
        hb1 = comp1.getHeartbeatInfo()
        hb2 = comp2.getHeartbeatInfo()

        for worker in hb1:
            if worker['worker_name'] == 'testWorker1':
                self.assertTrue(worker["cycle_time"] > 1.0)
            else:
                self.assertEqual(worker["cycle_time"], 0)
        self.assertItemsEqual([item["outcome"] for item in hb1], [None, None])
        self.assertItemsEqual([item["error_message"] for item in hb1], [None, None])

        self.assertEqual(round(hb2[0]["cycle_time"], 1), 1234.1)
        self.assertEqual(hb2[0]["outcome"], '100')
        self.assertEqual(hb2[0]["error_message"], None)

        # time to update workers with an error
        comp1.updateWorkerError("testWorker2", "BAD JOB!!!")
        hb1 = comp1.getHeartbeatInfo()
        for worker in hb1:
            if worker['worker_name'] == 'testWorker2':
                self.assertTrue(worker["last_error"] > int(time.time() - 10))
                self.assertEqual(worker["state"], "Error")
                self.assertEqual(worker["error_message"], "BAD JOB!!!")

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class JobCreatorTest(unittest.TestCase):
    """
    Test case for the JobCreator

    """

    sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also, create some dummy locations.
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()

        self.testInit.setSchema(customModules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'],
                                useDefault=False)
        self.couchdbname = "jobcreator_t"
        self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump")
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        locationAction = self.daoFactory(classname="Locations.New")
        for site in self.sites:
            locationAction.execute(siteName=site, pnn=site)

        # Create sites in resourceControl

        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName=site, pnn=site, ceName=site)
            resourceControl.insertThreshold(siteName=site, taskType='Processing', maxSlots=10000, pendingSlots=10000)

        self.resourceControl = resourceControl

        self._setup = True
        self._teardown = False

        self.testDir = self.testInit.generateWorkDir()
        self.cwd = os.getcwd()

        # Set heartbeat
        self.componentName = 'JobCreator'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """

        self.testInit.clearDatabase(modules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'])

        self.testInit.delWorkDir()

        self._teardown = True

        self.testInit.tearDownCouch()
        EmulatorSetup.deleteConfig(self.configFile)

        return

    def createJobCollection(self, name, nSubs, nFiles, workflowURL='test'):
        """
        _createJobCollection_

        Create a collection of jobs
        """

        myThread = threading.currentThread()

        testWorkflow = Workflow(spec=workflowURL, owner="mnorman",
                                name=name, task="/TestWorkload/ReReco")
        testWorkflow.create()

        for sub in range(nSubs):

            nameStr = '%s-%i' % (name, sub)

            myThread.transaction.begin()

            testFileset = Fileset(name=nameStr)
            testFileset.create()

            for f in range(nFiles):
                # pick a random site
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class JobSubmitterTest(unittest.TestCase):
    """
    _JobSubmitterTest_

    Test class for the JobSubmitterPoller
    """

    def setUp(self):
        """
        _setUp_

        Standard setup: Now with 100% more couch
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
                                       logger = myThread.logger,
                                       dbinterface = myThread.dbi)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        return

    def tearDown(self):
        """
        _tearDown_

        Standard tearDown
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        self.testInit.tearDownCouch()
        return

    def setResourceThresholds(self, site, **options):
        """
        _setResourceThresholds_

        Utility to set resource thresholds
        """
        if not options:
            options = {'state'        : 'Normal',
                       'runningSlots' : 10,
                       'pendingSlots' : 5,
                       'tasks' : ['Processing', 'Merge'],
                       'Processing' : {'pendingSlots' : 5,
                                       'runningSlots' : 10},
                       'Merge' : {'pendingSlots' : 2,
                                  'runningSlots' : 5}}

        resourceControl = ResourceControl()
        resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site),
                                   ceName = site, plugin = "MockPlugin", pendingSlots = options['pendingSlots'],
                                   runningSlots = options['runningSlots'], cmsName = site)
        for task in options['tasks']:
            resourceControl.insertThreshold(siteName = site, taskType = task,
                                            maxSlots = options[task]['runningSlots'],
                                            pendingSlots = options[task]['pendingSlots'])
        if options.get('state'):
            resourceControl.changeSiteState(site, options.get('state'))

        return

    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
                        bl = [], wl = [], taskType = 'Processing', name = None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()

        testWorkflow = Workflow(spec = workloadSpec, owner = "mnorman",
                                name = name, task = "basicWorkload/Production")
        testWorkflow.create()

        # Create subscriptions
        for _ in range(nSubs):

            name = makeUUID()

#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class BossAirTest(unittest.TestCase):
    """
    Tests for the BossAir prototype

    """

    sites = ['T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN', 'T2_US_Florida']

    def setUp(self):
        """
        setup for test.
        """

        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.tearDown()
        self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"],
                                useDefault = False)
        self.testInit.setupCouch("bossair_t/jobs", "JobDump")
        self.testInit.setupCouch("bossair_t/fwjrs", "FWJRDump")

        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")

        #Create sites in resourceControl
        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), cmsName = site,
                                       ceName = site, plugin = "CondorPlugin", pendingSlots = 1000,
                                       runningSlots = 2000)
            resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
                                            maxSlots = 1000, pendingSlots = 1000)
        resourceControl.insertSite(siteName = 'Xanadu', seName = 'se.Xanadu',cmsName = site,
                                   ceName = 'Xanadu', plugin = "TestPlugin")
        resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \
                                        maxSlots = 10000, pendingSlots = 10000)

        resourceControl.insertSite(siteName = 'jade-cms.hip.fi', seName = 'madhatter.csc.fi', cmsName = site,
                                   ceName = 'jade-cms.hip.fi', plugin = "ARCPlugin")
        resourceControl.insertThreshold(siteName = 'jade-cms.hip.fi', taskType = 'Processing', \
                                        maxSlots = 100, pendingSlots = 100)
        # using this for glite submissions
        resourceControl.insertSite(siteName = 'grid-ce-01.ba.infn.it', seName = 'storm-se-01.ba.infn.it', cmsName = site,
                                   ceName = 'grid-ce-01.ba.infn.it', plugin = 'gLitePlugin')
        resourceControl.insertThreshold(siteName = 'grid-ce-01.ba.infn.it', taskType = 'Processing', \
                                        maxSlots = 50, pendingSlots = 50)

        # Create user
        newuser = self.daoFactory(classname = "Users.New")
        newuser.execute(dn = "tapas", group_name = "phgroup", role_name = "cmsrole")


        # We actually need the user name
        self.user = getpass.getuser()

        # Change this to the working dir to keep track of error and log files from condor
        self.testInit.generateWorkDir()

        # Set heartbeat
        componentName = 'test'
        self.heartbeatAPI  = HeartbeatAPI(componentName)
        self.heartbeatAPI.registerComponent()
        componentName = 'JobTracker'
        self.heartbeatAPI2  = HeartbeatAPI(componentName)
        self.heartbeatAPI2.registerComponent()

        return

    def tearDown(self):
        """
        Database deletion
        """
        #self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])

        self.testInit.delWorkDir()

        self.testInit.tearDownCouch()

        return



    def getConfig(self):
        """
        _getConfig_

        Build a basic BossAir config
        """

        config = self.testInit.getConfiguration()

        config.section_("Agent")
        config.Agent.agentName  = 'testAgent'
        config.Agent.componentName = 'test'
        config.Agent.useHeartbeat = False
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class DBSUploadTest(unittest.TestCase):
    """
    TestCase for DBSUpload module

    Note:
      This fails if you use the in-memory syntax for sqlite
      i.e. (DATABASE = sqlite://)
    """

    _maxMessage = 10

    def setUp(self):
        """
        _setUp_

        setUp function for unittest

        """
        # Set constants
        self.couchDB = "config_test"
        self.configURL = "RANDOM;;URL;;NAME"
        self.configString = "This is a random string"

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.Agent.Database"], useDefault=False)
        self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(
            package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi
        )

        locationAction = self.bufferFactory(classname="DBSBufferFiles.AddLocation")
        locationAction.execute(siteName="se1.cern.ch")
        locationAction.execute(siteName="se1.fnal.gov")
        locationAction.execute(siteName="malpaquet")

        # Set heartbeat
        self.componentName = "JobSubmitter"
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        # Set up a config cache
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDB)
        configCache.createUserGroup(groupname="testGroup", username="testOps")
        self.testDir = self.testInit.generateWorkDir()

        psetPath = os.path.join(self.testDir, "PSet.txt")
        f = open(psetPath, "w")
        f.write(self.configString)
        f.close()

        configCache.addConfig(newConfig=psetPath, psetHash=None)
        configCache.save()
        self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB, configCache.getCouchID())

        return

    def tearDown(self):
        """
        _tearDown_

        tearDown function for unittest
        """

        self.testInit.clearDatabase(modules=["WMComponent.DBS3Buffer", "WMCore.Agent.Database"])

    def createConfig(self):
        """
        _createConfig_

        This creates the actual config file used by the component

        """
        config = Configuration()

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = "DBSUpload"
        config.Agent.useHeartbeat = False

        # Now the CoreDatabase information
        # This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.component_("DBSUpload")
        config.DBSUpload.pollInterval = 10
        config.DBSUpload.logLevel = "ERROR"
        config.DBSUpload.maxThreads = 1
        config.DBSUpload.namespace = "WMComponent.DBSUpload.DBSUpload"
        config.DBSUpload.componentDir = os.path.join(os.getcwd(), "Components")
        config.DBSUpload.workerThreads = 4

#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]

#.........这里部分代码省略.........

        -mnorman
        """
        return

    def initialization(self):
        """
        _initialization__

        Used the handle initializing the MsgService.  The MsgService
        is no longer used.

        Removed but not deleted, since all sorts of things call it
        """
        return

    def prepareToStart(self):
        """
        _prepareToStart_

        returns: Nothing

        Starts the initialization procedure. It is mainly an aggregation method
        so it can easily used in tests.
        """
        self.state = "initialize"
        self.initInThread()
        # note: every component gets a (unique) name:
        # self.config.Agent.componentName
        logging.info(">>>Registering Component - %s" % self.config.Agent.componentName)

        if getattr(self.config.Agent, "useHeartbeat", True):
            self.heartbeatAPI = HeartbeatAPI(self.config.Agent.componentName)
            self.heartbeatAPI.registerComponent()

        logging.info(">>>Starting initialization")

        logging.info(">>>Setting default transaction")
        myThread = threading.currentThread()

        self.preInitialization()

        if myThread.sql_transaction:
            myThread.transaction.begin()

        self.initialization()
        self.postInitialization()

        if myThread.sql_transaction:
            myThread.transaction.commit()

        logging.info(">>>Committing default transaction")

        logging.info(">>>Starting worker threads")
        myThread.workerThreadManager.resumeWorkers()

        logging.info(">>>Initialization finished!\n")
        # wait for messages
        self.state = "active"

    def prepareToStop(self, wait=False, stopPayload=""):
        """
        _stopComponent

        Stops the component, including all worker threads. Allows call from
        test framework

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class JobSubmitterTest(unittest.TestCase):
    """
    _JobSubmitterTest_

    Test class for the JobSubmitterPoller
    """

    def setUp(self):
        """
        _setUp_

        Standard setup: Now with 100% more couch
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(
            customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"]
        )
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
        self.baDaoFactory = DAOFactory(package="WMCore.BossAir", logger=myThread.logger, dbinterface=myThread.dbi)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = "JobSubmitter"
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        return

    def tearDown(self):
        """
        _tearDown_

        Standard tearDown
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        self.testInit.tearDownCouch()
        EmulatorSetup.deleteConfig(self.configFile)
        return

    def setResourceThresholds(self, site, **options):
        """
        _setResourceThresholds_

        Utility to set resource thresholds
        """
        if not options:
            options = {
                "state": "Normal",
                "runningSlots": 10,
                "pendingSlots": 5,
                "tasks": ["Processing", "Merge"],
                "Processing": {"pendingSlots": 5, "runningSlots": 10},
                "Merge": {"pendingSlots": 2, "runningSlots": 5},
            }

        resourceControl = ResourceControl()
        resourceControl.insertSite(
            siteName=site,
            pnn="se.%s" % (site),
            ceName=site,
            plugin="MockPlugin",
            pendingSlots=options["pendingSlots"],
            runningSlots=options["runningSlots"],
            cmsName=site,
        )
        for task in options["tasks"]:
            resourceControl.insertThreshold(
                siteName=site,
                taskType=task,
                maxSlots=options[task]["runningSlots"],
                pendingSlots=options[task]["pendingSlots"],
            )
        if options.get("state"):
            resourceControl.changeSiteState(site, options.get("state"))

        return

    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site, taskType="Processing", name=None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()

        testWorkflow = Workflow(spec=workloadSpec, owner="tapas", name=name, task="basicWorkload/Production")
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class JobSubmitterTest(unittest.TestCase):
    """
    Test class for the JobSubmitter

    """

    sites = ["T2_US_Florida", "T2_US_UCSD", "T2_TW_Taiwan", "T1_CH_CERN"]

    def setUp(self):
        """
        Standard setup: Now with 100% more couch
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setSchema(
            customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"],
            useDefault=False,
        )
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        locationAction = self.daoFactory(classname="Locations.New")
        locationSlots = self.daoFactory(classname="Locations.SetJobSlots")

        # We actually need the user name
        self.user = getpass.getuser()

        self.ceName = "127.0.0.1"

        # Create sites in resourceControl
        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(
                siteName=site,
                seName="se.%s" % (site),
                ceName=site,
                plugin="CondorPlugin",
                pendingSlots=10000,
                runningSlots=20000,
                cmsName=site,
            )
            resourceControl.insertThreshold(siteName=site, taskType="Processing", maxSlots=10000)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = "JobSubmitter"
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        return

    def tearDown(self):
        """
        Standard tearDown

        """
        self.testInit.clearDatabase(
            modules=["WMCore.WMBS", "WMCore.ResourceControl", "WMCore.BossAir", "WMCore.Agent.Database"]
        )
        self.testInit.delWorkDir()

        self.testInit.tearDownCouch()

        return

    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site=None, bl=[], wl=[], type="Processing"):
        """
        Creates a series of jobGroups for submissions

        """

        jobGroupList = []

        testWorkflow = Workflow(spec=workloadSpec, owner="mnorman", name=makeUUID(), task="basicWorkload/Production")
        testWorkflow.create()

        # Create subscriptions
        for i in range(nSubs):

            name = makeUUID()

            # Create Fileset, Subscription, jobGroup
            testFileset = Fileset(name=name)
            testFileset.create()
            testSubscription = Subscription(
                fileset=testFileset, workflow=testWorkflow, type=type, split_algo="FileBased"
            )
            testSubscription.create()

            testJobGroup = JobGroup(subscription=testSubscription)
            testJobGroup.create()

            # Create jobs
            self.makeNJobs(
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class JobSubmitterTest(EmulatedUnitTestCase):
    """
    _JobSubmitterTest_

    Test class for the JobSubmitterPoller
    """

    def setUp(self):
        """
        _setUp_

        Standard setup: Now with 100% more couch
        """
        super(JobSubmitterTest, self).setUp()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(
            customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
                                       logger=myThread.logger,
                                       dbinterface=myThread.dbi)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()
        self.configFile = EmulatorSetup.setupWMAgentConfig()
        config = self.getConfig()
        myThread.logdbClient = MockLogDB(config.General.central_logdb_url,
                                         config.Agent.hostName, logger=None)
        return

    def tearDown(self):
        """
        _tearDown_

        Standard tearDown
        """
        myThread = threading.currentThread()
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        self.testInit.tearDownCouch()
        EmulatorSetup.deleteConfig(self.configFile)
        myThread.logdbClient = None
        return

    def setResourceThresholds(self, site, **options):
        """
        _setResourceThresholds_

        Utility to set resource thresholds
        """
        if not options:
            options = {'state': 'Normal',
                       'runningSlots': 10,
                       'pendingSlots': 5,
                       'tasks': ['Processing', 'Merge'],
                       'Processing': {'pendingSlots': 5,
                                      'runningSlots': 10},
                       'Merge': {'pendingSlots': 2,
                                 'runningSlots': 5}}

        resourceControl = ResourceControl()
        resourceControl.insertSite(siteName=site, pnn='se.%s' % (site),
                                   ceName=site, plugin="MockPlugin", pendingSlots=options['pendingSlots'],
                                   runningSlots=options['runningSlots'], cmsName=site)
        for task in options['tasks']:
            resourceControl.insertThreshold(siteName=site, taskType=task,
                                            maxSlots=options[task]['runningSlots'],
                                            pendingSlots=options[task]['pendingSlots'])
        if options.get('state'):
            resourceControl.changeSiteState(site, options.get('state'))

        return

    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
                        taskType='Processing', name=None, wfPrio=1, changeState=None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        changeState is an instance of the ChangeState class to make job status changes
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Agent.HeartbeatAPI import HeartbeatAPI [as 别名]
# 或者: from WMCore.Agent.HeartbeatAPI.HeartbeatAPI import registerComponent [as 别名]
class DBSUploadTest(unittest.TestCase):
    """
    _DBSUploadTest_

    TestCase for DBSUpload module
    """

    _maxMessage = 10

    def setUp(self):
        """
        _setUp_

        setUp function for unittest
        """
        # Set constants
        self.couchDB      = "config_test"
        self.configURL    = "RANDOM;;URL;;NAME"
        self.configString = "This is a random string"

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules =
                                ["WMComponent.DBS3Buffer",
                                 'WMCore.Agent.Database'],
                                useDefault = False)
        self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
                                        logger = myThread.logger,
                                        dbinterface = myThread.dbi)
        self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                         logger = myThread.logger,
                                         dbinterface = myThread.dbi)

        locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
        locationAction.execute(siteName = "se1.cern.ch")
        locationAction.execute(siteName = "se1.fnal.gov")
        locationAction.execute(siteName = "malpaquet")

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI  = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        # Set up a config cache
        configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB)
        configCache.createUserGroup(groupname = "testGroup", username = 'testOps')
        self.testDir = self.testInit.generateWorkDir()

        psetPath = os.path.join(self.testDir, "PSet.txt")
        f = open(psetPath, 'w')
        f.write(self.configString)
        f.close()

        configCache.addConfig(newConfig = psetPath, psetHash = None)
        configCache.save()
        self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"],
                                         self.couchDB,
                                         configCache.getCouchID())
        return

    def tearDown(self):
        """
        _tearDown_

        tearDown function for unittest
        """

        self.testInit.clearDatabase()
        self.testInit.tearDownCouch()
        self.testInit.delWorkDir()
        return

    def createConfig(self):
        """
        _createConfig_

        This creates the actual config file used by the component

        """
        config = Configuration()

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat    = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


#.........这里部分代码省略.........