Python WMSpecGenerator.createReRecoSpec方法代码示例

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class WorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB, InboxDbName=self.globalQInboxDB, NegotiationTimeout=0)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "%s%s" % (self.specNamePrefix, (i + 1))
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testQueueElementProfile(self):
        self.createProfile("queueElementProfile.prof", self.multipleQueueWorkCall)

    def multipleQueueWorkCall(self):
        i = 0
        for wmspec in self.specs:
            i += 1
            self.globalQueue.queueWork(wmspec, self.specNamePrefix + str(i), "test_team")

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable. 

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True, 
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        #EmulatorSetup.deleteConfig(self.configFile)
        self.specGenerator.removeSpecs()
        

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
    def createWorkload(self):
        """
        Create a workload in order to test things

        """
        generator = WMSpecGenerator()
        workload = generator.createReRecoSpec("Tier1ReReco")
        return workload

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        EmulatorHelper.resetEmulators()
        self.specGenerator.removeSpecs()


    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        wqApi.defaultOptions =  {'reduce' : True, 'group' : True}
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        localQ = localQueue(DbName = 'local_workqueue_t',
                            QueueURL = self.testInit.couchUrl,
                            CacheDir = self.testInit.testDir,
                            ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName = 'workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        globalApi.defaultOptions =  {'reduce' : True, 'group' : True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0)
        localQ.processInboundWork(continuous = False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        #overwrite default - can't test with stale view
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class WorkQueueTest(EmulatedUnitTestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """

    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema,
                                useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        self.specGenerator.removeSpecs()
        super(WorkQueueTest, self).tearDown()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 339, 'request_name': specName}])
        # work still available, so no childQueue
        results = wqApi.getChildQueuesAndStatus()
        self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"])
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)

        results = wqApi.getChildQueuesAndPriority()
        resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"])
        self.assertItemsEqual(resultsPrio, [8000])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        localQ = localQueue(DbName='local_workqueue_t',
                            QueueURL=self.testInit.couchUrl,
                            CacheDir=self.testInit.testDir,
                            ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName='workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        globalApi.defaultOptions = {'reduce': True, 'group': True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(
            DbName=self.queueDB,
            InboxDbName=self.queueInboxDB,
            NegotiationTimeout=0,
            QueueURL="global.example.com",
            CacheDir=self.cacheDir,
        )

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i + 1)
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile("getWorkProfile.prof", self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in Globals.SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import createReRecoSpec [as 别名]
class RequestManager(dict):
    
    def __init__(self, *args, **kwargs):
        """
        all the private valuable is defined for test values
        """
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
        self.type = kwargs.setdefault("type", 'ReReco')
        if self.type not in ['ReReco', 'MonteCarlo']:
            raise TypeError, 'unknown request type %s' % self.type
        self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
        self.inputDataset = kwargs.setdefault('inputDataset', None)
        self.dbsUrl = kwargs.setdefault('dbsUrl', None)
        self.status = {}
        self.progress = {}
        self.msg = {}
        self.names = []
        import logging
        self['logger'] = logging
    
    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            if self.type == 'ReReco':
                specName = "ReRecoTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createReRecoSpec(specName, "file",
                                                             self.splitter,
                                                             self.inputDataset,
                                                             self.dbsUrl)
            elif self.type == 'MonteCarlo':
                specName = "MCTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createMCSpec(specName, "file",
                                                         self.splitter)
            self.names.append(specName)
            self.status[specName] = 'assigned'
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            #specName = "FakeProcessingSpec_%s" % self.count
            #specUrl =self.specGenerator.createProcessingSpec(specName, "file")
        
            self.count += 1
            # returns list of list(rquest name, spec url)
            return [[specName, specUrl],]
        else:
            return []
    
    def putWorkQueue(self, reqName, prodAgentUrl=None):
        self.status[reqName] = 'acquired'

    def reportRequestStatus(self, name, status):
        if status not in NextStatus[self.status[name]]:
            raise RuntimeError, "Invalid status move: %s" % status
        self.status[name] = status

    def reportRequestProgress(self, name, **args):
        self.progress.setdefault(name, {})
        self.progress[name].update(args)

    def sendMessage(self, request, msg):
        self.msg[request] = msg
        
    def _removeSpecs(self):
        """
        This is just for clean up not part of emulated function
        """
        self.specGenerator.removeSpecs()