Python TestInitCouchApp.setLogging方法代码示例

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable. 

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True, 
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        #EmulatorSetup.deleteConfig(self.configFile)
        self.specGenerator.removeSpecs()
        

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class RESTBaseUnitTest(unittest.TestCase):
    
    def setUp(self):
        # default set
        self.schemaModules = []
        self.initialize()
        if self.schemaModules:
            self.testInit = TestInitCouchApp(__file__)
            self.testInit.setLogging() # logLevel = logging.SQLDEBUG
            self.testInit.setDatabaseConnection(self.config.getDBUrl())
            self.testInit.setSchema(customModules = self.schemaModules,
                                    useDefault = False)
        
        self.rt = Root(self.config)
        self.rt.start(blocking=False)
        
    def tearDown(self):
        self.rt.stop()
        if self.schemaModules:
            self.testInit.clearDatabase()
        self.config = None
        
    
    def initialize(self):
        """
        i.e.
        
        self.config = DefaultConfig('WMCore.WebTools.RESTModel')
        self.config.setDBUrl("sqlite://")
        self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"]
        """
        
        message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set"
        raise NotImplementedError, message

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class T0RequestDBTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["T0Request"]
        self.testInit = TestInitCouchApp('RequestDBServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 't0_requsetdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
        self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        result =  self.requestWriter.insertGenericRequest(schema[0])

        self.assertEqual(len(result), 1, 'insert fail');
        
        result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")

        self.assertEqual(result, 'not allowed state assigned', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
                          'Error: document not found')
        
        allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",  
                         "Processing Done", "completed"]
        for state in allowedStates:
            self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
                          'OK')
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
                          'not allowed transition completed to Processing Done')  
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
                          'OK')  
        result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
        self.assertEqual(len(result), 1, "should be 1 but %s" % result)

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class FWJRDBAPITest(unittest.TestCase):
    """
    """

    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["FWJRDump"]
        self.testInit = TestInitCouchApp("FWJRAPITest")
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        dbName = "fwjrdump_t"
        self.testInit.setupCouch(dbName, *self.couchApps)
        fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.fwjrAPI = FWJRDBAPI(fwjrDBURL)
        self.fwjrAPI.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testFWJRDBAPI(self):

        jobID = 1
        retryCount = 0
        fwjrDocument = {
            "_id": "%s-%s" % (jobID, retryCount),
            "jobid": jobID,
            "retrycount": retryCount,
            "archivestatus": "ready",
            "fwjr": SAMPLE_FWJR,
            "jobtype": "Merge",
            "type": "fwjr",
        }
        workflow = SAMPLE_FWJR["task"].split("/")[1]

        self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), False)
        self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), True)
        self.assertEqual(self.fwjrAPI.getFWJRWithSkippedFiles()["rows"][0]["value"]["skippedFiles"], 2)

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class CouchSinkTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        dbName = "couch_sink"
        self.testInit.setupCouch(dbName)

        self.config = ConfigSection("couch")
        self.config.url = self.testInit.couchUrl
        self.config.database = self.testInit.couchDbName


    def tearDown(self):
        self.testInit.tearDownCouch()


    def testCouchSinkBasic(self):
        sink = CouchSink(self.config)
        docIds = []
        for i in range(10):
            a = Alert(Source = __file__, Level = i, Timestamp = time.time(),
                      Type = "Test")
            retVals = sink.send([a])
            # return value is following format:
            # [{'rev': '1-ba0a0903d4d6ddcbb85ff64d48d8be14', 'id': 'b7e8f807c96f572418b39422ccea252c'}]
            # just 1 item was added in the list of alerts, so retVal is also 1 item list
            # and CMSCouch call commitOne also returns a list - hence second nesting
            docIds.append(retVals[0][0]["id"])
        changes = sink.database.changes()
        self.assertEqual(len(changes[u"results"]), 10)
        self.assertEqual(changes[u"last_seq"], 10)

        for i in range(10, 20):
            a = Alert(Source = __file__, Level = i, Timestamp = time.time(),
                      Type = "Test")
            retVals = sink.send([a])
            # just 1 item was added in the list of alerts, so retVal is also 1 item list
            # and CMSCouch call commitOne also returns a list - hence second nesting
            docIds.append(retVals[0][0]["id"])
        changes = sink.database.changes()
        self.assertEqual(len(changes[u"results"]), 10)
        self.assertEqual(changes[u"last_seq"], 20)

        # check documents presence
        for id, level in zip(docIds, range(20)):
            doc = sink.database.document(id)
            self.assertEqual(doc["Level"], level)

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class RESTSinkTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        dbName = "alerts-rest_sink"
        self.testInit.setupCouch(dbName)
        
        self.config = ConfigSection("rest")
        self.config.uri = self.testInit.couchUrl + "/" + dbName


    def tearDown(self):
        self.testInit.tearDownCouch()        

        
    def testRESTSinkBasic(self):
        sink = RESTSink(self.config)
        docIds = []
        alerts = []
        for i in range(10):
            a = Alert(Source = __file__, Level = i, Timestamp = time.time(),
                      Type = "Test")
            alerts.append(a)
        retVal = sink.send(alerts)
        
        # return value is following format:
        # [{'rev': '1-ba0a0903d4d6ddcbb85ff64d48d8be14', 'id': 'b7e8f807c96f572418b39422ccea252c'}]
        # just 1 item was added in the list of alerts, so retVal is also 1 item list
        # and CMSCouch call commitOne also returns a list - hence second nesting            
        changes = sink._database.changes()
        self.assertEqual(len(changes[u"results"]), 10)
        self.assertEqual(changes[u"last_seq"], 10)        
        
        alerts = []
        for i in range(10, 20):
            a = Alert(Source = __file__, Level = i, Timestamp = time.time(),
                      Type = "Test")
            alerts.append(a)
        retVals = sink.send(alerts)
        
        changes = sink._database.changes()
        self.assertEqual(len(changes[u"results"]), 10)
        self.assertEqual(changes[u"last_seq"], 20)

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class FWJRDBAPITest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["FWJRDump"]
        self.testInit = TestInitCouchApp('FWJRAPITest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'fwjrdump_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.fwjrAPI= FWJRDBAPI(fwjrDBURL)
        self.fwjrAPI.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testFWJRDBAPI(self):
        
        jobID = 1
        retryCount = 0
        fwjrDocument = {"_id": "%s-%s" % (jobID, retryCount),
                        "jobid": jobID,
                        "retrycount": retryCount,
                        "archivestatus": "ready",
                        "fwjr": SAMPLE_FWJR,
                        "type": "fwjr"}
        self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]['id'], fwjrDocument['_id'])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")['rows'][0]['id'], fwjrDocument['_id'])      
        self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument['_id'])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")['rows'][0]['id'], fwjrDocument['_id'])

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class RESTBaseUnitTest(unittest.TestCase):

    def setUp(self, initRoot = True):
        # default set
        self.schemaModules = []

        self.initialize()
        if self.schemaModules:
            import warnings
            warnings.warn("use RESTAndCouchUnitTest instead", DeprecationWarning)
            from WMQuality.TestInitCouchApp import TestInitCouchApp
            self.testInit = TestInitCouchApp(__file__)
            self.testInit.setLogging() # logLevel = logging.SQLDEBUG
            self.testInit.setDatabaseConnection()
            self.testInit.setSchema(customModules = self.schemaModules,
                                    useDefault = False)
            # Now pull the dbURL from the factory
            # I prefer this method because the factory has better error handling
            # Also because then you know everything is the same
            myThread = threading.currentThread()
            self.config.setDBUrl(myThread.dbFactory.dburl)

        logging.info("This is our config: %s" % self.config)

        self.initRoot = initRoot
        if initRoot:
            self.rt = Root(self.config, testName=self._testMethodName)
            try:
                self.rt.start(blocking=False)
            except RuntimeError, e:
                # there appears to be worker threads from a previous test
                # hanging out. Try to slay them so that we can keep going
                print "Failed to load cherrypy with exception: %s\n" % e
                print "The threads are: \n%s\n" % threading.enumerate()
                print "The previous test was %s\n" % self.rt.getLastTest()
                print traceback.format_exc()
                self.rt.stop()
                raise e

        return

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class WorkloadSummary_t(unittest.TestCase):
    def setUp(self):
        """bootstrap tests"""
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        #self.testInit.setDatabaseConnection()
        #self.testInit.setSchema(customModules = ["WMCore.WMBS"],
        #                        useDefault = False)
        self.testInit.setupCouch("wmcore-workloadsummary", "WorkloadSummary")
        
        self.workload1 = newWorkload("WorkloadSummaryTest1")
        self.workload2 = newWorkload("WorkloadSummaryTest2")

        
    def tearDown(self):
        self.testInit.tearDownCouch()
        self.testInit.clearDatabase()
        pass    

    def testA(self):
        """
        register workloads in the couchapp
        and pulling back information from the views
        """
        populateWorkload(self.workload1, "evansde77")
        populateWorkload(self.workload2, "drsm79")
        
        summary1 = self.workload1.generateWorkloadSummary()
        summary2 = self.workload2.generateWorkloadSummary()
        
        summ1 = WorkloadSummary(self.workload1.name(), self.testInit.couchUrl, self.testInit.couchDbName, self.workload1)
        summ2 = WorkloadSummary(self.workload2.name(), self.testInit.couchUrl, self.testInit.couchDbName, self.workload2)
        summ1.create()
        summ2.create()
        
        summ1.addACDCCollection(makeUUID())
        for t in self.workload1.listAllTaskPathNames():
            fakeDoc = makeUUID()
            summ1.addACDCFileset(t, fakeDoc)

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class TestChangeState(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        
        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/Task"
        return

    def tearDown(self):
        """
        _tearDown_

        Cleanup the databases.
        """
        self.testInit.clearDatabase()
        self.testInit.tearDownCouch()
        return

    def testCheck(self):
        """
        This is the test class for function Check from module ChangeState
        """
        change = ChangeState(self.config, "changestate_t")

        # Run through all good state transitions and assert that they work
        for state in self.transitions.keys():
            for dest in self.transitions[state]:
                change.check(dest, state)
        dummystates = ['dummy1', 'dummy2', 'dummy3', 'dummy4']

        # Then run through some bad state transistions and assertRaises(AssertionError)
        for state in self.transitions.keys():
            for dest in dummystates:
                self.assertRaises(AssertionError, change.check, dest, state)
        return

    def testRecordInCouch(self):
        """
        _testRecordInCouch_

        Verify that jobs, state transitions and fwjrs are recorded correctly.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname = "Locations.New")
        locationAction.execute("site1", seName = "somese.cern.ch")

        testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
                                name = "wf001", task = self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name = "TestFileset")
        testFileset.create()
        testSubscription = Subscription(fileset = testFileset,
                                        workflow = testWorkflow,
                                        split_algo = "FileBased")
        testSubscription.create()

        testFileA = File(lfn = "SomeLFNA", events = 1024, size = 2048,
                         locations = set(["somese.cern.ch"]))
        testFileB = File(lfn = "SomeLFNB", events = 1025, size = 2049,
                         locations = set(["somese.cern.ch"]))
        testFileA.create()
        testFileB.create()

        testFileset.addFile(testFileA)
        testFileset.addFile(testFileB)
        testFileset.commit()

        splitter = SplitterFactory()
        jobFactory = splitter(package = "WMCore.WMBS",
                              subscription = testSubscription)
        jobGroup = jobFactory(files_per_job = 1)[0]

        assert len(jobGroup.jobs) == 2, \
               "Error: Splitting should have created two jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "sfoulkes"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Merge"
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class ReportTest(unittest.TestCase):
    """
    _ReportTest_

    Unit tests for the Report class.
    """
    def setUp(self):
        """
        _setUp_

        Figure out the location of the XML report produced by CMSSW.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase = True)
        self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")

        self.xmlPath = os.path.join(getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
        self.badxmlPath = os.path.join(getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
        self.skippedFilesxmlPath = os.path.join(getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml")
        self.skippedAllFilesxmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml")
        self.fallbackXmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml")
        self.testDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Cleanup the databases.
        """
        self.testInit.tearDownCouch()
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        return

    def verifyInputData(self, report):
        """
        _verifyInputData_

        Verify that the input file in the Report class matches the input file in
        the XML generated by CMSSW.
        """
        inputFiles = report.getInputFilesFromStep("cmsRun1")

        assert len(inputFiles) == 1, \
               "Error: Wrong number of input files."
        assert inputFiles[0]["lfn"] == "/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \
               "Error: Wrong LFN on input file."
        assert inputFiles[0]["pfn"] == "dcap://cmsdca.fnal.gov:24137/pnfs/fnal.gov/usr/cms/WAX/11/store/data/BeamCommissioning09/MinimumBias/RAW/v1/000/122/023/142F3F42-C5D6-DE11-945D-000423D94494.root", \
               "Error: Wrong PFN on input file."

        inputRun = list(inputFiles[0]["runs"])
        assert len(inputRun) == 1, \
               "Error: Wrong number of runs in input."
        assert inputRun[0].run == 122023, \
               "Error: Wrong run number on input file."
        assert len(inputRun[0].lumis) == 1, \
               "Error: Wrong number of lumis in input file."
        assert 215 in inputRun[0].lumis, \
               "Error: Input file is missing lumis."

        assert inputFiles[0]["events"] == 2, \
               "Error: Wrong number of events in input file."
        assert inputFiles[0]["size"] == 0, \
               "Error: Wrong size in input file."

        assert inputFiles[0]["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
               "Error: Catalog on input file is wrong."
        assert inputFiles[0]["guid"] == "142F3F42-C5D6-DE11-945D-000423D94494", \
               "Error: GUID of input file is wrong."

        return

    def verifyRecoOutput(self, report):
        """
        _verifyRecoOutput_

        Verify that all the metadata in the RECO output module is correct.
        """
        outputFiles = report.getFilesFromOutputModule("cmsRun1", "outputRECORECO")

        assert len(outputFiles) == 1, \
               "Error: Wrong number of output files."
        assert outputFiles[0]["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root", \
               "Error: Wrong LFN on output file: %s" % outputFiles[0]["lfn"]
        assert outputFiles[0]["pfn"] == "outputRECORECO.root", \
               "Error: Wrong PFN on output file."

        outputRun = list(outputFiles[0]["runs"])
        assert len(outputRun) == 1, \
               "Error: Wrong number of runs in output."
        assert outputRun[0].run == 122023, \
               "Error: Wrong run number on output file."
        assert len(outputRun[0].lumis) == 1, \
#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class CouchappTest(unittest.TestCase):

    def setUp(self):
        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.databaseName = "couchapp_t_0"
        self.testInit.setupCouch(self.databaseName, "WorkloadSummary")
        self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")

        # Setup config for couch connections
        config = self.testInit.getConfiguration()
        config.section_("JobStateMachine")
        config.JobStateMachine.couchDBName  = self.databaseName

        # Create couch server and connect to databases
        self.couchdb      = CouchServer(config.JobStateMachine.couchurl)
        self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % config.JobStateMachine.couchDBName)
        self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % config.JobStateMachine.couchDBName)

        # Create changeState
        self.changeState = ChangeState(config)
        self.config      = config

        # Create testDir
        self.testDir = self.testInit.generateWorkDir()

        return

    def tearDown(self):

        self.testInit.clearDatabase(modules = ["WMCore.WMBS"])
        self.testInit.delWorkDir()
        #self.testInit.tearDownCouch()
        return


    def createWorkload(self, workloadName = 'Test', emulator = True):
        """
        _createTestWorkload_

        Creates a test workload for us to run on, hold the basic necessities.
        """

        workload = testWorkload("Tier1ReReco")

        taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        workload.save(workloadName)

        return workload

    def createTestJobGroup(self, name = "TestWorkthrough",
                           specLocation = "spec.xml", error = False,
                           task = "/TestWorkload/ReReco", nJobs = 10):
        """
        _createTestJobGroup_

        Generate a test WMBS JobGroup with real FWJRs
        """

        myThread = threading.currentThread()

        testWorkflow = Workflow(spec = specLocation, owner = "Simon",
                                name = name, task = task)
        testWorkflow.create()

        testWMBSFileset = Fileset(name = name)
        testWMBSFileset.create()

        testFileA = File(lfn = makeUUID(), size = 1024, events = 10)
        testFileA.addRun(Run(10, *[12312]))
        testFileA.setLocation('malpaquet')

        testFileB = File(lfn = makeUUID(), size = 1024, events = 10)
        testFileB.addRun(Run(10, *[12312]))
        testFileB.setLocation('malpaquet')

        testFileA.create()
        testFileB.create()

        testWMBSFileset.addFile(testFileA)
        testWMBSFileset.addFile(testFileB)
        testWMBSFileset.commit()
        testWMBSFileset.markOpen(0)

        testSubscription = Subscription(fileset = testWMBSFileset,
                                        workflow = testWorkflow)
        testSubscription.create()

        testJobGroup = JobGroup(subscription = testSubscription)
        testJobGroup.create()

#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class RetryManagerTest(unittest.TestCase):
    """
    TestCase for TestRetryManager module
    """
    def setUp(self):
        """
        setup for test.
        """
        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.testInit.setupCouch("retry_manager_t/jobs", "JobDump")
        self.testInit.setupCouch("retry_manager_t/fwjrs", "FWJRDump")

        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
        self.setJobTime = self.daofactory(classname = "Jobs.SetStateTime")
        self.increaseRetry = self.daofactory(classname = "Jobs.IncrementRetry")
        self.testDir = self.testInit.generateWorkDir()
        self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.nJobs = 10
        return

    def tearDown(self):
        """
        Database deletion
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        self.testInit.tearDownCouch()
        EmulatorSetup.deleteConfig(self.configFile)
        return

    def getConfig(self):
        """
        _getConfig_

        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        config.component_("RetryManager")
        config.RetryManager.logLevel     = 'DEBUG'
        config.RetryManager.namespace    = 'WMComponent.RetryManager.RetryManager'
        config.RetryManager.pollInterval = 10
        # These are the cooloff times for the RetryManager, the times it waits
        # Before attempting resubmission
        config.RetryManager.section_("DefaultRetryAlgo")
        config.RetryManager.DefaultRetryAlgo.section_("default")
        config.RetryManager.DefaultRetryAlgo.default.coolOffTime  = {'create': 120, 'submit': 120, 'job': 120}
        # Path to plugin directory
        config.RetryManager.pluginPath   = 'WMComponent.RetryManager.PlugIns'
        config.RetryManager.WMCoreBase   = WMCore.WMBase.getWMBASE()
        config.RetryManager.componentDir = os.path.join(os.getcwd(), 'Components')

        # ErrorHandler
        # Not essential, but useful for ProcessingAlgo
        config.component_("ErrorHandler")
        config.ErrorHandler.maxRetries = 5

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = "retry_manager_t"

        return config

    def createTestJobGroup(self, nJobs, subType = "Processing", retryOnce = False):
        """
        _createTestJobGroup_

        Creates a group of several jobs
        """
        testWorkflow = Workflow(spec = "spec.xml", owner = "Simon",
                                name = makeUUID(), task="Test")
        testWorkflow.create()

        testWMBSFileset = Fileset(name = "TestFileset")
        testWMBSFileset.create()
        testSubscription = Subscription(fileset = testWMBSFileset,
                                        workflow = testWorkflow,
                                        type = subType)
        testSubscription.create()

        testJobGroup = JobGroup(subscription = testSubscription)
        testJobGroup.create()

#.........这里部分代码省略.........

# 需要导入模块: from WMQuality.TestInitCouchApp import TestInitCouchApp [as 别名]
# 或者: from WMQuality.TestInitCouchApp.TestInitCouchApp import setLogging [as 别名]
class JobSubmitterTest(unittest.TestCase):
    """
    _JobSubmitterTest_

    Test class for the JobSubmitterPoller
    """

    def setUp(self):
        """
        _setUp_

        Standard setup: Now with 100% more couch
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
                                       logger = myThread.logger,
                                       dbinterface = myThread.dbi)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        return

    def tearDown(self):
        """
        _tearDown_

        Standard tearDown
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        self.testInit.tearDownCouch()
        return

    def setResourceThresholds(self, site, **options):
        """
        _setResourceThresholds_

        Utility to set resource thresholds
        """
        if not options:
            options = {'state'        : 'Normal',
                       'runningSlots' : 10,
                       'pendingSlots' : 5,
                       'tasks' : ['Processing', 'Merge'],
                       'Processing' : {'pendingSlots' : 5,
                                       'runningSlots' : 10},
                       'Merge' : {'pendingSlots' : 2,
                                  'runningSlots' : 5}}

        resourceControl = ResourceControl()
        resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site),
                                   ceName = site, plugin = "MockPlugin", pendingSlots = options['pendingSlots'],
                                   runningSlots = options['runningSlots'], cmsName = site)
        for task in options['tasks']:
            resourceControl.insertThreshold(siteName = site, taskType = task,
                                            maxSlots = options[task]['runningSlots'],
                                            pendingSlots = options[task]['pendingSlots'])
        if options.get('state'):
            resourceControl.changeSiteState(site, options.get('state'))

        return

    def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
                        bl = [], wl = [], taskType = 'Processing', name = None):
        """
        _createJobGroups_

        Creates a series of jobGroups for submissions
        """

        jobGroupList = []

        if name is None:
            name = makeUUID()

        testWorkflow = Workflow(spec = workloadSpec, owner = "mnorman",
                                name = name, task = "basicWorkload/Production")
        testWorkflow.create()

        # Create subscriptions
        for _ in range(nSubs):

            name = makeUUID()

#.........这里部分代码省略.........