Python WorkQueue.getJobsByStatus方法代码示例

# 需要导入模块: from WMCore.Services.WorkQueue.WorkQueue import WorkQueue [as 别名]
# 或者: from WMCore.Services.WorkQueue.WorkQueue.WorkQueue import getJobsByStatus [as 别名]
    def testCompletedWorkflow(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist':['T2_XX_SiteA']})

        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 339, 'request_name': specName}])

        results = wqApi.getJobsByStatus()
        self.assertEqual(results['Available']['sum_jobs'], 339)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set([item['priority'] for item in results.get('Available')])
        self.assertItemsEqual(resultsPrio, [8000])
        resultsJobs = sum([item['sum_jobs'] for item in results.get('Available') if item['priority'] == 8000])
        self.assertTrue(resultsJobs, 339)
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)
        data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
                                 {'startkey': [specName], 'endkey': [specName, {}],
                                  'reduce': False})
        elements = [x['id'] for x in data.get('rows', [])]
        wqApi.updateElements(*elements, Status='Canceled')
        # load this view once again to make sure it will be updated in the next assert..
        data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
                                 {'startkey': [specName], 'endkey': [specName, {}],
                                  'reduce': False})
        self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set([item['priority'] for item in results.get('Canceled')])
        self.assertItemsEqual(resultsPrio, [8000])

# 需要导入模块: from WMCore.Services.WorkQueue.WorkQueue import WorkQueue [as 别名]
# 或者: from WMCore.Services.WorkQueue.WorkQueue.WorkQueue import getJobsByStatus [as 别名]

#.........这里部分代码省略.........
            agentInfo = self.collectAgentInfo()
            self.checkProxyLifetime(agentInfo)

            timeSpent, wmbsInfo, _ = self.collectWMBSInfo()
            wmbsInfo['total_query_time'] = int(timeSpent)
            agentInfo["WMBS_INFO"] = wmbsInfo
            logging.info("WMBS data collected in: %d secs", timeSpent)

            if not hasattr(self.config, "Tier0Feeder"):
                # Tier0 Agent doesn't have LQ.
                timeSpent, localWQInfo, _ = self.collectWorkQueueInfo()
                localWQInfo['total_query_time'] = int(timeSpent)
                agentInfo["LocalWQ_INFO"] = localWQInfo
                logging.info("Local WorkQueue data collected in: %d secs", timeSpent)

            uploadTime = int(time.time())
            self.uploadAgentInfoToCentralWMStats(agentInfo, uploadTime)

            # save locally json file as well
            with open(self.jsonFile, 'w') as outFile:
                json.dump(agentInfo, outFile, indent=2)

        except Exception as ex:
            logging.exception("Error occurred, will retry later.\nDetails: %s", str(ex))

    @timeFunction
    def collectWorkQueueInfo(self):
        """
        Collect information from local workqueue database
        :return:
        """
        results = {}

        results['workByStatus'] = self.workqueueDS.getJobsByStatus()
        results['workByStatusAndPriority'] = self.workqueueDS.getJobsByStatusAndPriority()

        elements = self.workqueueDS.getElementsByStatus(['Available', 'Acquired'])
        uniSites, posSites = getGlobalSiteStatusSummary(elements, dataLocality=True)
        results['uniqueJobsPerSite'] = uniSites
        results['possibleJobsPerSite'] = posSites

        return results

    def collectCouchDBInfo(self):

        couchInfo = {'name': 'CouchServer', 'status': 'ok', 'error_message': ""}

        if self.skipReplicationCheck:
            # skipping the check this round set if False so it can be checked next round.
            self.skipReplicationCheck = False
            return couchInfo

        for rp in self.replicatorDocs:
            cInfo = self.localCouchMonitor.checkCouchServerStatus(rp['source'],
                                                                  rp['target'], checkUpdateSeq=False)
            if cInfo['status'] != 'ok':
                couchInfo['status'] = 'error'
                couchInfo['error_message'] = cInfo['error_message']

        return couchInfo

    def collectAgentInfo(self):
        """
        Monitors the general health of the agent, as:
          1. status of the agent processes
          2. status of the agent threads based on the database info

# 需要导入模块: from WMCore.Services.WorkQueue.WorkQueue import WorkQueue [as 别名]
# 或者: from WMCore.Services.WorkQueue.WorkQueue.WorkQueue import getJobsByStatus [as 别名]

#.........这里部分代码省略.........
        """
        try:
            agentInfo = self.collectAgentInfo()
            self.checkCredLifetime(agentInfo, "proxy")
            self.checkCredLifetime(agentInfo, "certificate")

            timeSpent, wmbsInfo, _ = self.collectWMBSInfo()
            wmbsInfo['total_query_time'] = int(timeSpent)
            agentInfo["WMBS_INFO"] = wmbsInfo
            logging.info("WMBS data collected in: %d secs", timeSpent)

            if not self.isT0agent:
                timeSpent, localWQInfo, _ = self.collectWorkQueueInfo()
                localWQInfo['total_query_time'] = int(timeSpent)
                agentInfo["LocalWQ_INFO"] = localWQInfo
                logging.info("Local WorkQueue data collected in: %d secs", timeSpent)

            self.uploadAgentInfoToCentralWMStats(agentInfo)

            self.buildMonITDocs(agentInfo)

        except Exception as ex:
            logging.exception("Error occurred, will retry later.\nDetails: %s", str(ex))

    @timeFunction
    def collectWorkQueueInfo(self):
        """
        Collect information from local workqueue database
        :return:
        """
        results = {}
        wqStates = ['Available', 'Acquired']

        results['workByStatus'] = self.workqueueDS.getJobsByStatus()
        results['workByStatusAndPriority'] = self.workqueueDS.getJobsByStatusAndPriority()

        elements = self.workqueueDS.getElementsByStatus(wqStates)
        uniSites, posSites = getGlobalSiteStatusSummary(elements, status=wqStates, dataLocality=True)
        results['uniqueJobsPerSite'] = uniSites
        results['possibleJobsPerSite'] = posSites

        return results

    def collectCouchDBInfo(self):

        couchInfo = {'name': 'CouchServer', 'status': 'ok', 'error_message': ""}

        if self.skipReplicationCheck:
            # skipping the check this round set if False so it can be checked next round.
            self.skipReplicationCheck = False
            return couchInfo

        for rp in self.replicatorDocs:
            cInfo = self.localCouchMonitor.checkCouchServerStatus(rp['source'],
                                                                  rp['target'], checkUpdateSeq=False)
            if cInfo['status'] != 'ok':
                couchInfo['status'] = 'error'
                couchInfo['error_message'] = cInfo['error_message']

        return couchInfo

    def collectAgentInfo(self):
        """
        Monitors the general health of the agent, as:
          1. status of the agent processes
          2. status of the agent threads based on the database info