Python RequestDBWriter.getRequestByStatus方法代码示例

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import getRequestByStatus [as 别名]
class T0RequestDBTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["T0Request"]
        self.testInit = TestInitCouchApp('RequestDBServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 't0_requsetdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
        self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        result =  self.requestWriter.insertGenericRequest(schema[0])

        self.assertEqual(len(result), 1, 'insert fail');
        
        result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")

        self.assertEqual(result, 'not allowed state assigned', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
                          'Error: document not found')
        
        allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",  
                         "Processing Done", "completed"]
        for state in allowedStates:
            self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
                          'OK')
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
                          'not allowed transition completed to Processing Done')  
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
                          'OK')  
        result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
        self.assertEqual(len(result), 1, "should be 1 but %s" % result)

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import getRequestByStatus [as 别名]

#.........这里部分代码省略.........
            mergeSub.load()
            self.stateMap['Harvesting'].append(mergeSub)

        harvestingWorkflow = Workflow(name=workflowName, task=harvestingTask)
        harvestingWorkflow.load()
        harvestingFileset = Fileset(name='/PromptReco_Run195360_Cosmics/Reco/RecoMergewrite_DQM/merged-MergedDQM')
        harvestingFileset.load()
        harvestingSub = Subscription(harvestingFileset, harvestingWorkflow)
        harvestingSub.load()
        self.stateMap['Processing Done'].append(harvestingSub)

        return

    def verifyStateTransitions(self, transitionMethod='markFinished', transitionTrigger=True):
        """
        _verifyStateTransitions_

        Utility method which goes through the list of states in self.orderedStates and
        finishes the tasks that demand a state transition in each step. This according
        to the defined transition method and trigger.
        It verifies that the request document in WMStats is moving according to the transitions
        """

        for idx in range(0, len(self.orderedStates) * 2):
            nextState = self.orderedStates[idx / 2]
            if (idx / 2) == 0:
                currentState = 'Closed'
            else:
                currentState = self.orderedStates[idx / 2 - 1]
            if idx % 2 == 0:
                for transitionObject in self.stateMap[nextState][:-1]:
                    method = getattr(transitionObject, transitionMethod)
                    method(transitionTrigger)
                self.plugin([], self.requestDBWriter, self.requestDBWriter)
                currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState])
                nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState])
                self.assertEqual(len(currentStateWorkflows), 1, 'Workflow moved incorrectly from %s' % currentState)
                self.assertEqual(len(nextStateWorkflows), 0, 'Workflow moved incorrectly to %s' % nextState)
            else:
                transitionObject = self.stateMap[nextState][-1]
                method = getattr(transitionObject, transitionMethod)
                method(transitionTrigger)
                self.plugin([], self.requestDBWriter, self.requestDBWriter)
                currentStateWorkflows = self.requestDBWriter.getRequestByStatus([currentState])
                nextStateWorkflows = self.requestDBWriter.getRequestByStatus([nextState])
                self.assertEqual(len(currentStateWorkflows), 0,
                                 'Workflow did not move correctly from %s' % currentState)
                self.assertEqual(len(nextStateWorkflows), 1, 'Workflow did not move correctly to %s' % nextState)
        return

    def testA_RepackStates(self):
        """
        _testA_RepackStates_

        Setup an environment with a Repack workflow
        and traverse through the different states.
        Check that the transitions are sane.
        """
        # Set the environment
        self.setupRepackWorkflow()
        self.plugin = Tier0Plugin()

        # Verify the transitions
        self.verifyStateTransitions('markOpen', False)

        return

    def testB_ExpressStates(self):
        """
        _testB_ExpressStates_

        Setup an environment with a Express workflow
        and traverse through the different states.
        Check that the transitions are sane.
        """
        # Set the environment
        self.setupExpressWorkflow()
        self.plugin = Tier0Plugin()

        # Verify the transitions
        self.verifyStateTransitions()

        return

    def testC_PromptRecoStates(self):
        """
        _testC_PromptRecoStates_

        Setup an environment with a PromptReco workflow
        and traverse through the different states.
        Check that the transitions are sane.
        """
        # Set the environment
        self.setupPromptRecoWorkflow()
        self.plugin = Tier0Plugin()

        # Verify the transitions
        self.verifyStateTransitions()

        return

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import getRequestByStatus [as 别名]

#.........这里部分代码省略.........
        myThread.transaction.begin()

        # Get the subscriptions that are now finished and mark them as such
        logging.info("Polling for finished subscriptions")
        finishedSubscriptions = self.daoFactory(classname="Subscriptions.MarkNewFinishedSubscriptions")
        finishedSubscriptions.execute(self.stateID, timeOut=self.timeout)
        logging.info("Finished subscriptions updated")

        myThread.transaction.commit()

        return

    def getFinishedWorkflows(self):
        """
        1. Get finished workflows (a finished workflow is defined in Workflow.GetFinishedWorkflows)
        2. Get finished workflows with logCollect and Cleanup only.
        3. combined those and make return
           finishedwfs - without LogCollect and CleanUp task
           finishedwfsWithLogCollectAndCleanUp - including LogCollect and CleanUp task
        """

        finishedWorkflowsDAO = self.daoFactory(classname="Workflow.GetFinishedWorkflows")
        finishedwfs = finishedWorkflowsDAO.execute()
        finishedLogCollectAndCleanUpwfs = finishedWorkflowsDAO.execute(onlySecondary=True)
        finishedwfsWithLogCollectAndCleanUp = {}
        for wf in finishedLogCollectAndCleanUpwfs:
            if wf in finishedwfs:
                finishedwfsWithLogCollectAndCleanUp[wf] = finishedwfs[wf]
        return (finishedwfs, finishedwfsWithLogCollectAndCleanUp)

    def killCondorJobsByWFStatus(self, statusList):
        if isinstance(statusList, basestring):
            statusList = [statusList]
        reqNames = self.centralCouchDBWriter.getRequestByStatus(statusList)
        logging.info("There are %d requests in %s status in central couch.", len(reqNames), statusList)
        self.workQueue.killWMBSWorkflows(reqNames)
        return reqNames

    def completeTasks(self, finishedwfs):
        """
        _completeTasks_

        This method will call several auxiliary methods to do the following:

        1. Notify the WorkQueue about finished subscriptions
        2. mark workflow as completed in the dbsbuffer_workflow table
        """
        if not finishedwfs:
            return

        logging.info("Found %d candidate workflows for completing:", len(finishedwfs))
        completedWorkflowsDAO = self.dbsDaoFactory(classname="UpdateWorkflowsToCompleted")

        centralCouchAlive = True
        try:
            self.killCondorJobsByWFStatus(["force-complete", "aborted"])
        except Exception as ex:
            centralCouchAlive = False
            logging.error("we will try again when remote couch server comes back\n%s", str(ex))

        if centralCouchAlive:
            logging.info("Marking subscriptions as Done ...")
            for workflow in finishedwfs:
                try:
                    # Notify the WorkQueue, if there is one
                    if self.workQueue is not None:

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import getRequestByStatus [as 别名]

#.........这里部分代码省略.........
        finishedWorkflowsDAO = self.daoFactory(classname = "Workflow.GetFinishedWorkflows")
        finishedwfs = finishedWorkflowsDAO.execute()
        finishedLogCollectAndCleanUpwfs = finishedWorkflowsDAO.execute(onlySecondary=True)
        finishedwfsWithLogCollectAndCleanUp = {}
        for wf in finishedLogCollectAndCleanUpwfs:
            if wf in finishedwfs:
                finishedwfsWithLogCollectAndCleanUp[wf] = finishedwfs[wf]
        return (finishedwfs, finishedwfsWithLogCollectAndCleanUp)
        
    def completeTasks(self, finishedwfs):
        """
        _completeTasks_

        This method will call several auxiliary methods to do the following:
        
        1. Notify the WorkQueue about finished subscriptions
        2. update dbsbuffer_workflow table with finished subscription
        """


        #Only delete those where the upload and notification succeeded
        logging.info("Found %d candidate workflows for completing: %s" % (len(finishedwfs),finishedwfs.keys()))
        # update the completed flag in dbsbuffer_workflow table so blocks can be closed
        # create updateDBSBufferWorkflowComplete DAO
        if len(finishedwfs) == 0:
            return
        
        completedWorkflowsDAO = self.dbsDaoFactory(classname = "UpdateWorkflowsToCompleted")
        
        centralCouchAlive = True
        try:
            #TODO: need to enable when reqmgr2 -wmstats is ready
            #abortedWorkflows = self.reqmgrCouchDBWriter.getRequestByStatus(["aborted"], format = "dict");
            abortedWorkflows = self.centralCouchDBWriter.getRequestByStatus(["aborted"])
            logging.info("There are %d requests in 'aborted' status in central couch." % len(abortedWorkflows))
            forceCompleteWorkflows = self.centralCouchDBWriter.getRequestByStatus(["force-complete"])
            logging.info("List of 'force-complete' workflows in central couch: %s" % forceCompleteWorkflows)
            
        except Exception as ex:
            centralCouchAlive = False
            logging.error("we will try again when remote couch server comes back\n%s" % str(ex))
        
        if centralCouchAlive:
            for workflow in finishedwfs:
                try:
                    #Notify the WorkQueue, if there is one
                    if self.workQueue != None:
                        subList = []
                        logging.info("Marking subscriptions as Done ...")
                        for l in finishedwfs[workflow]["workflows"].values():
                            subList.extend(l)
                        self.notifyWorkQueue(subList)
                    
                    #Now we know the workflow as a whole is gone, we can delete the information from couch
                    if not self.useReqMgrForCompletionCheck:
                        self.requestLocalCouchDB.updateRequestStatus(workflow, "completed")
                        logging.info("status updated to completed %s" % workflow)
    
                    if workflow in abortedWorkflows:
                        #TODO: remove when reqmgr2-wmstats deployed
                        newState = "aborted-completed"
                    elif workflow in forceCompleteWorkflows:
                        newState = "completed"
                    else:
                        newState = None