Python WMStatsWriter.insertRequest方法代码示例

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import insertRequest [as 别名]
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import insertRequest [as 别名]
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
    """
    If clone is True, the function is called on a cloned request in which
    case no modification of the reqSchema shall happen and should be checked in
    as is.
    
    """
    try:
        request = buildWorkloadForRequest(typename = reqSchema["RequestType"], 
                                          schema = reqSchema)
    except WMSpecFactoryException as ex:
        logging.error(traceback.format_exc())
        raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
    
    helper = WMWorkloadHelper(request['WorkloadSpec'])
    
    # update request as well for wmstats update
    # there is a better way to do this (passing helper to request but make sure all the information is there) 
    request["Campaign"] = helper.getCampaign()

    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request['OutputDatasets']:
                request['OutputDatasets'].append(ds)
    #TODO: need to update output dataset by Task for task chain requests
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)            
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema['RequestType'])
    except CheckIn.RequestCheckInError as ex:
        raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
        
    # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
    # metadata above is what is saved into couch to represent a request document.
    # Number of request arguments on a corresponding couch document
    # is not set, has default null/None values, update those accordingly now.
    # It's a mess to have two mutually inconsistent database backends.
    # Not easy to handle this earlier since couch is stored first and
    # some parameters are worked out later when storing into Oracle.
    reqDetails = requestDetails(request["RequestName"])
    # couchdb request parameters which are null at the injection time and remain so
    paramsToUpdate = ["RequestStatus",
                      "RequestSizeFiles",
                      "AcquisitionEra",
                      "RequestWorkflow",
                      "RequestType",
                      "RequestStatus",
                      "RequestPriority",
                      "Requestor",
                      "Group",
                      "SizePerEvent",
                      "PrepID",
                      "RequestNumEvents",
                      "ProcessingString",
                      "ProcessingVersion",
                      ]
    
    couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) 
        
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400, "Creating request failed, could not update WMStats.")

    return request

# 需要导入模块: from WMCore.Services.WMStats.WMStatsWriter import WMStatsWriter [as 别名]
# 或者: from WMCore.Services.WMStats.WMStatsWriter.WMStatsWriter import insertRequest [as 别名]
                      "Requestor",
                      "Group",
                      "SizePerEvent",
                      "PrepID",
                      "RequestNumEvents",
                      ]
    
    couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields) 
        
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400, "Creating request failed, could not update WMStats.")

    return request
    
        
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
    """
    Handles the submission of requests.
    
    """
    # make sure no extra spaces snuck in
    for k, v in reqInputArgs.iteritems():
        if isinstance(v, str):