Python WMSpecGenerator.removeSpecs方法代码示例

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class WorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB, InboxDbName=self.globalQInboxDB, NegotiationTimeout=0)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "%s%s" % (self.specNamePrefix, (i + 1))
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testQueueElementProfile(self):
        self.createProfile("queueElementProfile.prof", self.multipleQueueWorkCall)

    def multipleQueueWorkCall(self):
        i = 0
        for wmspec in self.specs:
            i += 1
            self.globalQueue.queueWork(wmspec, self.specNamePrefix + str(i), "test_team")

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable. 

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True, 
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        #EmulatorSetup.deleteConfig(self.configFile)
        self.specGenerator.removeSpecs()
        

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        EmulatorHelper.resetEmulators()
        self.specGenerator.removeSpecs()


    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        wqApi.defaultOptions =  {'reduce' : True, 'group' : True}
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        localQ = localQueue(DbName = 'local_workqueue_t',
                            QueueURL = self.testInit.couchUrl,
                            CacheDir = self.testInit.testDir,
                            ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName = 'workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        globalApi.defaultOptions =  {'reduce' : True, 'group' : True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0)
        localQ.processInboundWork(continuous = False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        #overwrite default - can't test with stale view
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class WorkQueueTest(EmulatedUnitTestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """

    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema,
                                useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        self.specGenerator.removeSpecs()
        super(WorkQueueTest, self).tearDown()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 339, 'request_name': specName}])
        # work still available, so no childQueue
        results = wqApi.getChildQueuesAndStatus()
        self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"])
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)

        results = wqApi.getChildQueuesAndPriority()
        resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"])
        self.assertItemsEqual(resultsPrio, [8000])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        localQ = localQueue(DbName='local_workqueue_t',
                            QueueURL=self.testInit.couchUrl,
                            CacheDir=self.testInit.testDir,
                            ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName='workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        globalApi.defaultOptions = {'reduce': True, 'group': True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class TestChangeState(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/ReReco1"
        self.specGen = WMSpecGenerator()
        self.specUrl = self.specGen.createProcessingSpec("TestWorkflow", returnType="file")
        return

    def tearDown(self):
        """
        _tearDown_

        Cleanup the databases.
        """
        self.testInit.clearDatabase()
        self.testInit.tearDownCouch()
        self.specGen.removeSpecs()
        return

    def testCheck(self):
        """
        This is the test class for function Check from module ChangeState
        """
        change = ChangeState(self.config, "changestate_t")

        # Run through all good state transitions and assert that they work
        for state in self.transitions.keys():
            for dest in self.transitions[state]:
                change.check(dest, state)
        dummystates = ['dummy1', 'dummy2', 'dummy3', 'dummy4']

        # Then run through some bad state transistions and assertRaises(AssertionError)
        for state in self.transitions.keys():
            for dest in dummystates:
                self.assertRaises(AssertionError, change.check, dest, state)
        return

    def testRecordInCouch(self):
        """
        _testRecordInCouch_

        Verify that jobs, state transitions and fwjrs are recorded correctly.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        testFileA = File(lfn="SomeLFNA", events=1024, size=2048,
                         locations=set(["T2_CH_CERN"]))
        testFileB = File(lfn="SomeLFNB", events=1025, size=2049,
                         locations=set(["T2_CH_CERN"]))
        testFileA.create()
        testFileB.create()

        testFileset.addFile(testFileA)
        testFileset.addFile(testFileB)
        testFileset.commit()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 2, \
               "Error: Splitting should have created two jobs."

        testJobA = jobGroup.jobs[0]
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(
            DbName=self.queueDB,
            InboxDbName=self.queueInboxDB,
            NegotiationTimeout=0,
            QueueURL="global.example.com",
            CacheDir=self.cacheDir,
        )

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i + 1)
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile("getWorkProfile.prof", self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in Globals.SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})

# 需要导入模块: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator import WMSpecGenerator [as 别名]
# 或者: from WMQuality.Emulators.WMSpecGenerator.WMSpecGenerator.WMSpecGenerator import removeSpecs [as 别名]
class RequestManager(dict):
    
    def __init__(self, *args, **kwargs):
        """
        all the private valuable is defined for test values
        """
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
        self.type = kwargs.setdefault("type", 'ReReco')
        if self.type not in ['ReReco', 'MonteCarlo']:
            raise TypeError, 'unknown request type %s' % self.type
        self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
        self.inputDataset = kwargs.setdefault('inputDataset', None)
        self.dbsUrl = kwargs.setdefault('dbsUrl', None)
        self.status = {}
        self.progress = {}
        self.msg = {}
        self.names = []
        import logging
        self['logger'] = logging
    
    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            if self.type == 'ReReco':
                specName = "ReRecoTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createReRecoSpec(specName, "file",
                                                             self.splitter,
                                                             self.inputDataset,
                                                             self.dbsUrl)
            elif self.type == 'MonteCarlo':
                specName = "MCTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createMCSpec(specName, "file",
                                                         self.splitter)
            self.names.append(specName)
            self.status[specName] = 'assigned'
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            #specName = "FakeProcessingSpec_%s" % self.count
            #specUrl =self.specGenerator.createProcessingSpec(specName, "file")
        
            self.count += 1
            # returns list of list(rquest name, spec url)
            return [[specName, specUrl],]
        else:
            return []
    
    def putWorkQueue(self, reqName, prodAgentUrl=None):
        self.status[reqName] = 'acquired'

    def reportRequestStatus(self, name, status):
        if status not in NextStatus[self.status[name]]:
            raise RuntimeError, "Invalid status move: %s" % status
        self.status[name] = status

    def reportRequestProgress(self, name, **args):
        self.progress.setdefault(name, {})
        self.progress[name].update(args)

    def sendMessage(self, request, msg):
        self.msg[request] = msg
        
    def _removeSpecs(self):
        """
        This is just for clean up not part of emulated function
        """
        self.specGenerator.removeSpecs()