Python RequestDBWriter.updateRequestProperty方法代码示例

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import updateRequestProperty [as 别名]
class StepChainParentageFixTask(CherryPyPeriodicTask):
    """
    Upldates StepChain parentage periodically
    """

    def __init__(self, rest, config):

        super(StepChainParentageFixTask, self).__init__(config)
        self.reqmgrDB = RequestDBWriter(config.reqmgrdb_url)
        self.dbsSvc = DBS3Reader(config.dbs_url, logger=self.logger)
        self.statusToCheck = ["announced", "normal-archived"]

    def setConcurrentTasks(self, config):
        """
        sets the list of functions which
        """
        self.concurrentTasks = [{'func': self.fixStepChainParentage, 'duration': config.parentageFixDuration}]

    def fixStepChainParentage(self, config):
        """
        Look through the stepchain workflows with ParentageResolved flag is False.
        Fix the StepChain parentage and update the ParentageResolved flag to True
        """
        self.logger.info("Updating parentage for StepChain workflows for %s", self.statusToCheck)
        childDatasets = set()
        requests = set()
        requestsByChildDataset = {}
        for status in self.statusToCheck:
            reqByChildDS= getChildDatasetsForStepChainMissingParent(self.reqmgrDB, status)
            childDatasets = childDatasets.union(set(reqByChildDS.keys()))
            # We need to just get one of the StepChain workflow if multiple workflow contains the same datasets. (i.e. ACDC)
            requestsByChildDataset.update(reqByChildDS)

            for wfs in reqByChildDS.values():
                requests = requests.union(wfs)

        failedRequests = set()
        totalChildDS = len(childDatasets)
        fixCount = 0
        for childDS in childDatasets:
            start = int(time.time())
            failedBlocks = self.dbsSvc.fixMissingParentageDatasets(childDS, insertFlag=True)
            end = int(time.time())
            timeTaken = end - start
            if failedBlocks:
                self.logger.warning("Failed to fix the parentage for %s will be retried: time took: %s (sec)",
                                 failedBlocks, timeTaken)
                failedRequests = failedRequests.union(requestsByChildDataset[childDS])
            else:
                fixCount += 1
                self.logger.info("Fixed %s parentage: %s out of %s datasets. time took: %s (sec)",
                                 childDS, fixCount, totalChildDS, timeTaken)

        requestsToUpdate = requests - failedRequests

        for request in requestsToUpdate:
            self.reqmgrDB.updateRequestProperty(request, {"ParentageResolved": True})

        self.logger.info("Total %s requests' ParentageResolved flag is set to True", len(requestsToUpdate))
        self.logger.info("Total %s requests will be retried next cycle: %s", len(failedRequests), failedRequests)

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import updateRequestProperty [as 别名]

#.........这里部分代码省略.........

    def _combine_request(self, request_info, requestAgentUrl, cache):
        keys = {}
        requestAgentUrlList = []
        for row in requestAgentUrl["rows"]:
            request = row["key"][0]
            if not keys[request]:
                keys[request] = []
            keys[request].append(row["key"][1])

        for request in request_info:
            for agentUrl in keys[request]:
                requestAgentUrlList.append([request, agentUrl])

        return requestAgentUrlList

    def _retrieveResubmissionChildren(self, request_name):

        result = self.reqmgr_db.loadView('ReqMgr', 'childresubmissionrequests', keys=[request_name])['rows']
        childrenRequestNames = []
        for child in result:
            childrenRequestNames.append(child['id'])
            childrenRequestNames.extend(self._retrieveResubmissionChildren(child['id']))
        return childrenRequestNames

    def _handleNoStatusUpdate(self, workload, request_args):
        """
        only few values can be updated without state transition involved
        currently 'RequestPriority' and 'total_jobs', 'input_lumis', 'input_events', 'input_num_files'
        """
        if 'RequestPriority' in request_args:
            # must update three places: GQ elements, workload_cache and workload spec
            self.gq_service.updatePriority(workload.name(), request_args['RequestPriority'])
            report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args)
            workload.setPriority(request_args['RequestPriority'])
            workload.saveCouchUrl(workload.specUrl())
        elif "total_jobs" in request_args:
            # only GQ update this stats
            # request_args should contain only 4 keys 'total_jobs', 'input_lumis', 'input_events', 'input_num_files'}
            report = self.reqmgr_db_service.updateRequestStats(workload.name(), request_args)
        else:
            raise InvalidSpecParameterValue("can't update value without state transition: %s" % request_args)
        
        return report

    def _handleAssignmentApprovedTransition(self, workload, request_args, dn):
        report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)
        return report
        
    def _handleAssignmentStateTransition(self, workload, request_args, dn):
        
        req_status = request_args["RequestStatus"]
        if req_status == "assigned" and not request_args.get('Team', '').strip():
            raise InvalidSpecParameterValue("Team must be set during workflow assignment: %s" % request_args)
            
        if ('SoftTimeout' in request_args) and ('GracePeriod' in request_args):
            request_args['SoftTimeout'] = int(request_args['SoftTimeout'])
            #TODO: not sure why GracePeriod when passed from web ingerface but convert here
            request_args['GracePeriod'] = int(request_args['GracePeriod'])
            request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']
        
        #Only allow extra value update for assigned status
        cherrypy.log("INFO: Assign request, input args: %s ..." % request_args)
        try:
            workload.updateArguments(request_args)
        except Exception as ex:

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import updateRequestProperty [as 别名]
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'wmstats_t'
        self.testInit.setupCouch(dbName, "WMStats")
        reqDBName = "reqmgrdb_t"
        self.testInit.setupCouch(reqDBName, "ReqMgr")
        wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
        self.reqDBWriter = RequestDBWriter(reqDBURL)
        self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
        self.wmstatsReader.defaultStale = {}
        self.wmstatsReader.reqDB.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        
        result = self.reqDBWriter.insertGenericRequest(schema[0])
        self.assertEquals(result[0]['ok'], True, 'insert fail')
        
        result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed")
        self.assertEquals(result, 'OK', 'update fail')
        
        result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned") 
        self.assertEquals(result,'Error: document not found')
        
        result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']})
        self.assertEquals(result, 'OK', 'update fail')
        
        result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']})                  
        self.assertEquals(result, 'Error: document not found')
        
        totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5}
        result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
        self.assertEquals(result, 'OK', 'update fail')
        
        result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
        self.assertEquals(result, 'OK', 'update fail')
        
        result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats)
        self.assertEquals(result, 'Error: document not found')
        
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        properties = {"RequestPriority": spec1.priority(),
                      'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
                      'OutputDatasets': spec1.listOutputDatasets()}
        result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties)
        self.assertEquals(result, 'OK', 'update fail')
        
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        properties = {"RequestPriority": spec2.priority(),
                      'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
                      'OutputDatasets': spec2.listOutputDatasets()}
        result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties)
        self.assertEquals(result, 'Error: document not found')

        requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True)
        self.assertEquals(requests.keys(), [schema[0]['RequestName']])
        
        requestCollection = RequestInfoCollection(requests)
        result = requestCollection.getJSONData()
        self.assertEquals(result.keys(), [schema[0]['RequestName']])
        
        requests = self.wmstatsReader.getActiveData()
        self.assertEquals(requests.keys(), [schema[0]['RequestName']])
        requests = self.wmstatsReader.getRequestByStatus(["failed"])
        self.assertEquals(requests.keys(), [schema[0]['RequestName']])
        
        requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName'])
        self.assertEquals(requests.keys(), [schema[0]['RequestName']])

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import updateRequestProperty [as 别名]

#.........这里部分代码省略.........
            request_args['HardTimeout'] = request_args['SoftTimeout'] + request_args['GracePeriod']

        if 'RequestPriority' in request_args:
            self.gq_service.updatePriority(workload.name(), request_args['RequestPriority'])

        if "total_jobs" in request_args:
            # only GQ update this stats
            # request_args should contain only 4 keys 'total_jobs', 'input_lumis', 'input_events', 'input_num_files'}
            report = self.reqmgr_db_service.updateRequestStats(workload.name(), request_args)
        # if is not just updating status
        else:
            req_status = request_args.get("RequestStatus", None)

            if len(request_args) >= 1 and req_status == None:
                try:
                    workload.updateArguments(request_args)
                except Exception as ex:
                    msg = traceback.format_exc()
                    cherrypy.log("Error for request args %s: %s" % (request_args, msg))
                    raise InvalidSpecParameterValue(str(ex))

                # trailing / is needed for the savecouchUrl function
                workload.saveCouch(self.config.couch_host, self.config.couch_reqmgr_db)

            elif (req_status in ["closed-out"  "announced"]) and request_args.get("cascade", False):
                cascade_list = self._retrieveResubmissionChildren(workload.name)
                for req_name in cascade_list:
                    report = self.reqmgr_db_service.updateRequestStatus(req_name, req_status)

            # If it is aborted or force-complete transition call workqueue to cancel the request
            else:
                if req_status == "aborted" or req_status == "force-complete":
                    self.gq_service.cancelWorkflow(workload.name())
                report = self.reqmgr_db_service.updateRequestProperty(workload.name(), request_args, dn)

        if report == 'OK':
            return {workload.name(): "OK"}
        else:
            return {workload.name(): "ERROR"}

    @restcall(formats=[('application/json', JSONFormat())])
    def put(self, workload_pair_list):
        "workloadPairList is a list of tuple containing (workload, requeat_args)"
        report = []
        for workload, request_args in workload_pair_list:
            result = self._updateRequest(workload, request_args)
            report.append(result)
        return report

    @restcall(formats=[('application/json', JSONFormat())])
    def delete(self, request_name):
        cherrypy.log("INFO: Deleting request document '%s' ..." % request_name)
        try:
            self.reqmgr_db.delete_doc(request_name)
        except CouchError as ex:
            msg = "ERROR: Delete failed."
            cherrypy.log(msg + " Reason: %s" % ex)
            raise cherrypy.HTTPError(404, msg)
            # TODO
        # delete should also happen on WMStats
        cherrypy.log("INFO: Delete '%s' done." % request_name)

    def _update_additional_request_args(self, workload, request_args):
        """
        add to request_args properties which is not initially set from user.
        This data will put in to couchdb.

# 需要导入模块: from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter [as 别名]
# 或者: from WMCore.Services.RequestDB.RequestDBWriter.RequestDBWriter import updateRequestProperty [as 别名]
class RequestDBTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["ReqMgr"]
        self.testInit = TestInitCouchApp('RequestDBServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'requsetdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.requestWriter = RequestDBWriter(reqDBURL)
        self.requestReader = RequestDBReader(reqDBURL)
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema(3)
        result =  self.requestWriter.insertGenericRequest(schema[0])

        self.assertEqual(len(result), 1, 'insert fail');
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'Error: document not found')
        result = self.requestWriter.updateRequestProperty(schema[0]['RequestName'], 
                                                                   {'Teams': ['teamA']})
        self.assertEqual(self.requestWriter.updateRequestProperty(schema[0]['RequestName'], 
                                                                   {'Teams': ['teamA']}), 'OK', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestProperty("not_exist_schema", {'Teams': 'teamA'}),
                          'Error: document not found')
        
        result = self.requestReader.getRequestByNames([schema[0]['RequestName']])
        self.assertEqual(len(result), 1, "should be 1")
        result = self.requestReader.getRequestByStatus(["failed"], False, 1)
        self.assertEqual(len(result), 1, "should be 1")
        
        result = self.requestReader.getStatusAndTypeByRequest([schema[0]['RequestName']])
        self.assertEqual(result[schema[0]['RequestName']][0], 'failed', "should be failed")
        
        result =  self.requestWriter.insertGenericRequest(schema[1])
        time.sleep(2)
        result =  self.requestWriter.insertGenericRequest(schema[2])
        endTime = int(time.time()) - 1
        result = self.requestReader.getRequestByStatusAndEndTime("new", False, endTime)
        self.assertEqual(len(result), 1, "should be 1")
        endTime = int(time.time()) + 1
        result = self.requestReader.getRequestByStatusAndEndTime("new", False, endTime)
        self.assertEqual(len(result), 2, "should be 2")