Python TestInitCouchApp.TestInitCouchApp类代码示例

class ReproducibleSeedingTests(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
        self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache")
        self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl)

        self.documentId = None

    def tearDown(self):
        self.testInit.tearDownCouch()
        return


    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
                                     CouchDBName = self.testInit.couchDbName,
                                     ConfigCacheDoc = document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
        self.failUnless(seed1 != None)

class AnalysisCollectionService_t(unittest.TestCase):
    """
    Unit tests for AnalysisCollectionService
    """

    def setUp(self):
        """Set up test couch instance"""
        self.dbsURL  = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
        self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO'

        """Set up couch test environment"""
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC")

        self.acService = AnalysisCollectionService(url=self.testInit.couchUrl, database = self.testInit.couchDbName)
        self.collection = self.acService.createCollection('ewv-testcoll', 'ewv2', 'Analysis')


    def tearDown(self):
        """Clean up couch instance"""
        self.testInit.tearDownCouch()


    def testCreateFilesetFromDBS(self):
        """Test creating an analysis fileset from DBS"""

        rangesMask = Mask()
        rangesMask.addRunWithLumiRanges(run=1, lumiList=[[1, 9], [12, 12], [31, 31], [38, 39], [49, 49], [51, 52], [64, 65], [82, 82], [92, 98]])

        fs, fl = self.acService.createFilesetFromDBS(self.collection, filesetName='test_fs',
                                                     dbsURL=self.dbsURL, dataset=self.dataset, mask=rangesMask)

        self.assertTrue(fl['_id'])
        self.assertEqual(len(fl['files']), 21)

class T0RequestDBTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["T0Request"]
        self.testInit = TestInitCouchApp('RequestDBServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 't0_requsetdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
        self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        result =  self.requestWriter.insertGenericRequest(schema[0])

        self.assertEqual(len(result), 1, 'insert fail');
        
        result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")

        self.assertEqual(result, 'not allowed state assigned', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
                          'Error: document not found')
        
        allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",  
                         "Processing Done", "completed"]
        for state in allowedStates:
            self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
                          'OK')
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
                          'not allowed transition completed to Processing Done')  
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
                          'OK')  
        result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
        self.assertEqual(len(result), 1, "should be 1 but %s" % result)

class Interface_t(unittest.TestCase):

    def setUp(self):
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser")

        #create a test owner for the collections in this test
        self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName)
        self.owner1.connect()
        self.owner1.create()

        self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName)
        self.owner2.connect()
        self.owner2.create()

        #self.url = os.getenv("COUCHURL", "http://127.0.0.1:5984")
        #self.database = "awesome_acdc"

    def tearDown(self):
        self.testInit.tearDownCouch()

    def testA(self):
        """ make some documents and own them"""
        guInt = Interface(self.testInit.couchUrl, self.testInit.couchDbName)


        #create a couple of docs
        couch = Database(self.testInit.couchDbName, self.testInit.couchUrl)
        for x in range(10):
            doc = Document("document%s" % x, {"Test Data": [1,2,3,4] })
            couch.queue(doc)
        couch.commit()

        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 0)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)

        guInt.callUpdate("ownthis","document1", group = self.owner1.group.name, user = self.owner1.name)

        self.failUnless("document1" in guInt.documentsOwned(self.owner1.group.name, self.owner1.name))
        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)

        guInt.callUpdate("ownthis","document2", group = self.owner2.group.name, user = self.owner2.name)

        self.failUnless("document2" in guInt.documentsOwned(self.owner2.group.name, self.owner2.name))
        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 1)


        guInt.callUpdate("newgroup", "group-DataOps", group = "DataOps")

        self.failUnless(couch.documentExists("group-DataOps") )

        guInt.callUpdate("newuser", "user-damason", group = "DataOps", user = "damason")

        self.failUnless(couch.documentExists("user-damason") )

 def couchapp(self, couchappName):
     """
     Drop database in CouchDB.
     Reload (push) the couchapp couchappName (likely to be "OpsClipboard").
     
     """
     print "Pushing couchapp '%s' ..." % couchappName
     # here probably need to specify the couchUrl, otherwise default env COUCHURL
     # will be taken (as described above)
     testInit = TestInitCouchApp(__file__, dropExistingDb=True)
     # should now drop the CouchDB db, create a new one and push the couchapp
     testInit.setupCouch(self.dbName, couchappName)

class FWJRDBAPITest(unittest.TestCase):
    """
    """

    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["FWJRDump"]
        self.testInit = TestInitCouchApp("FWJRAPITest")
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        dbName = "fwjrdump_t"
        self.testInit.setupCouch(dbName, *self.couchApps)
        fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.fwjrAPI = FWJRDBAPI(fwjrDBURL)
        self.fwjrAPI.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testFWJRDBAPI(self):

        jobID = 1
        retryCount = 0
        fwjrDocument = {
            "_id": "%s-%s" % (jobID, retryCount),
            "jobid": jobID,
            "retrycount": retryCount,
            "archivestatus": "ready",
            "fwjr": SAMPLE_FWJR,
            "jobtype": "Merge",
            "type": "fwjr",
        }
        workflow = SAMPLE_FWJR["task"].split("/")[1]

        self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), False)
        self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), True)
        self.assertEqual(self.fwjrAPI.getFWJRWithSkippedFiles()["rows"][0]["value"]["skippedFiles"], 2)

class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')

class RESTBaseUnitTest(unittest.TestCase):
    
    def setUp(self):
        # default set
        self.schemaModules = []
        self.initialize()
        if self.schemaModules:
            self.testInit = TestInitCouchApp(__file__)
            self.testInit.setLogging() # logLevel = logging.SQLDEBUG
            self.testInit.setDatabaseConnection(self.config.getDBUrl())
            self.testInit.setSchema(customModules = self.schemaModules,
                                    useDefault = False)
        
        self.rt = Root(self.config)
        self.rt.start(blocking=False)
        
    def tearDown(self):
        self.rt.stop()
        if self.schemaModules:
            self.testInit.clearDatabase()
        self.config = None
        
    
    def initialize(self):
        """
        i.e.
        
        self.config = DefaultConfig('WMCore.WebTools.RESTModel')
        self.config.setDBUrl("sqlite://")
        self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"]
        """
        
        message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set"
        raise NotImplementedError, message

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.
        """
        logging.basicConfig()
        self.logger = logging.getLogger('LogDBTest')
#        self.logger.setLevel(logging.DEBUG)

        self.schema = []
        self.couchApps = ["LogDB"]
        self.testInit = TestInitCouchApp('LogDBTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'logdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        url = "%s/%s" % (self.testInit.couchUrl, dbName)

        identifier = 'agentname'
        self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test")
        identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
        self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.report = LogDBReport(self.agent_inst)

    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")        

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.couchServer = CouchServer(dburl = os.getenv("COUCHURL"))
        self.config = Configuration()
        self.config.component_("JobStateMachine")
        self.config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        return

    def setUp(self):
        """
        _setUp_

        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
        self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")        
        os.environ["COUCHDB"] = "wmbshelper_t"
        self.testInit.setSchema(customModules = ["WMCore.WMBS",
                                                 "WMComponent.DBSBuffer.Database",
                                                 "WMCore.BossAir",
                                                 "WMCore.ResourceControl"],
                                useDefault = False)
        
        self.workDir = self.testInit.generateWorkDir()
        
        self.wmspec = self.createWMSpec()
        self.topLevelTask = getFirstTask(self.wmspec)
        self.inputDataset = self.topLevelTask.inputDataset()
        self.dataset = self.topLevelTask.getInputDatasetPath()
        self.dbs = MockDBSReader(self.inputDataset.dbsurl)
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = threading.currentThread().logger,
                                     dbinterface = threading.currentThread().dbi)
        return

    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        
        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/Task"
        return

    def setUp(self):
        """
        _setUp_

        Figure out the location of the XML report produced by CMSSW.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")

        self.xmlPath = os.path.join(getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
        self.badxmlPath = os.path.join(getTestBase(),
                                       "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
        self.skippedFilesxmlPath = os.path.join(getTestBase(),
                                                "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml")
        self.skippedAllFilesxmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml")
        self.fallbackXmlPath = os.path.join(getTestBase(),
                                            "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml")
        self.twoFileFallbackXmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
        self.pileupXmlPath = os.path.join(getTestBase(),
                                          "WMCore_t/FwkJobReport_t/CMSSWPileup.xml")

        self.testDir = self.testInit.generateWorkDir()
        return

class ConfigurationTest(unittest.TestCase):
    """
    test case for Configuration object

    """
    def setUp(self):
        """set up"""
        self.testInit = TestInit(__file__)
        self.testDir  = self.testInit.generateWorkDir()
        self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir
        self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir
        self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir


    def tearDown(self):
        """clean up"""
        self.testInit.delWorkDir()


    def testA(self):
        """ctor"""
        try:
            config = Configuration()
        except Exception, ex:
            msg = "Failed to instantiate Configuration\n"
            msg += str(ex)
            self.fail(msg)

    def setUp(self):
        """
        setup for test.
        """
        super(JobArchiverTest, self).setUp()

        myThread = threading.currentThread()
        super(JobArchiverTest, self).setUp()
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        # self.tearDown()
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
        self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")

        self.daofactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")

        self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)

        self.nJobs = 10

        self.configFile = EmulatorSetup.setupWMAgentConfig()

        return