Python Database.updateDocument方法代码示例

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def changePriority(requestName, priority, wmstatUrl = None):
    """
    Changes the priority that's stored in the workload.
    Takes the current priority stored in the workload and adds
    to it the input priority value. 
    
    """
    request = requestDetails(requestName)
    # change in Oracle
    newPrior = int(priority)
    ChangeState.changeRequestPriority(requestName, newPrior)
    # change in workload (spec)
    helper = loadWorkload(request)
    helper.data.request.priority = newPrior
    saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
    # change priority in CouchDB
    couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
    fields = {"RequestPriority": newPrior}
    couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields)
    # push the change to the WorkQueue
    response = ProdManagement.getProdMgr(requestName)
    if response == [] or response[0] is None or response[0] == "":
        # Request must not be assigned yet, we are safe here
        return
    workqueue = WorkQueue.WorkQueue(response[0])
    workqueue.updatePriority(requestName, priority)
    return

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def main():
    print "Looking for problematic inbox elements..."
    problemRequests = getProblematicRequests()
    print "Found %d bad elements:" % len(problemRequests)
    if not problemRequests:
        print "Nothing to fix, contact a developer if the problem persists..."
        return 0
    for request in problemRequests:
        print request["RequestName"]
    var = raw_input("Can we close these for new data in inbox elements: Y/N\n")
    if var == "Y":
        print "Updating them in global inbox, you need a WMAgent proxy for this."
        inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
        for request in problemRequests:
            inboxDB.document(request._id)
            inboxDB.updateDocument(request._id, 'WorkQueue', 'in-place', fields={'OpenForNewData': false})
        print "Done with the deletions, this should fix the problem."
        return 0
    else:
        var = raw_input("Then can we delete these inbox elements: Y/N\n")
        if var == "Y":
            print "Deleting them from the global inbox, you need a WMAgent proxy for this."
            inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
            for request in problemRequests:
                inboxDB.delete_doc(request._id, request.rev)
            print "Done with the deletions, this should fix the problem."
            return 0
        else:
            print "Doing nothing as you commanded..."
        return 0

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
    """
    _changeRequestStatus_

    Basic API to change a request to a new state, also
    includes optional priority change for the request

    - *requestName* : name of the request to be modified
    - *newState*    : name of the new status for the request
    - *priority* : optional integer priority
    
    Apparently when changing request state (on assignment page),
    it's possible to change priority at one go. Hence the argument is here.

    """
    # MySQL/Oracle
    factory = DBConnect.getConnection()
    reqId = getRequestID(factory, requestName)
    changeRequestIDStatus(reqId, newState, priority)

    # CouchDB
    # have to first get information where the request Couch document is,
    # extracting the information from reqmgr_request.workflow table field
    reqData = factory(classname="Request.Get").execute(reqId)
    # this would be something like this:
    # http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
    wfUrl = reqData["workflow"]
    # cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
    couchUrl = wfUrl.replace("/" + requestName + "/spec", "")
    couchDbName = couchUrl[couchUrl.rfind("/") + 1 :]
    # cut off database name from the URL
    url = couchUrl.replace("/" + couchDbName, "")
    couchDb = Database(couchDbName, url)
    fields = {"RequestStatus": newState}
    couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody=True)

    # TODO: should we make this mendatory?
    if wmstatUrl:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.updateRequestStatus(requestName, newState)

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def changePriority(requestName, priority, wmstatUrl = None):
    """
    Changes the priority that's stored in the workload.
    Takes the current priority stored in the workload and adds
    to it the input priority value. 
    
    """
    request = requestDetails(requestName)
    # change in Oracle
    newPrior = int(priority)
    ChangeState.changeRequestPriority(requestName, newPrior)
    # change in workload (spec)
    helper = loadWorkload(request)
    helper.data.request.priority = newPrior
    saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
    # change priority in CouchDB
    couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
    fields = {"RequestPriority": newPrior}
    couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody = True)
    # push the change to the WorkQueue
    gqURL = "%s/workqueue" % request["CouchURL"]
    workqueue = WorkQueue.WorkQueue(gqURL)
    workqueue.updatePriority(requestName, priority)
    return

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]

#.........这里部分代码省略.........
            int(kwargs.get("MaxMergeSize", 4294967296)),
            int(kwargs.get("MaxMergeEvents", 50000)),
        )
        helper.setupPerformanceMonitoring(
            kwargs.get("MaxRSS", None),
            kwargs.get("MaxVSize", None),
            kwargs.get("SoftTimeout", None),
            kwargs.get("GracePeriod", None),
        )

        # Check whether we should check location for the data
        helper.setTrustLocationFlag(
            inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
            pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)),
        )
        helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))

        # Set phedex subscription information
        custodialList = kwargs.get("CustodialSites", [])
        nonCustodialList = kwargs.get("NonCustodialSites", [])
        autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
        for site in autoApproveList:
            if site.endswith("_MSS"):
                raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
        subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
        if subscriptionPriority not in ["Low", "Normal", "High"]:
            raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
        custodialType = kwargs.get("CustodialSubType", "Replica")
        if custodialType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
        nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
        if nonCustodialType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
        if "CustodialGroup" in kwargs and not isinstance(kwargs["CustodialGroup"], basestring):
            raise cherrypy.HTTPError(400, "Invalid CustodialGroup format %s" % kwargs["CustodialGroup"])
        if "NonCustodialGroup" in kwargs and not isinstance(kwargs["NonCustodialGroup"], basestring):
            raise cherrypy.HTTPError(400, "Invalid NonCustodialGroup format %s" % kwargs["NonCustodialGroup"])
        if "DeleteFromSource" in kwargs and not isinstance(kwargs["DeleteFromSource"], bool):
            raise cherrypy.HTTPError(400, "Invalid DeleteFromSource format %s" % kwargs["DeleteFromSource"])

        helper.setSubscriptionInformationWildCards(
            wildcardDict=self.wildcardSites,
            custodialSites=custodialList,
            nonCustodialSites=nonCustodialList,
            autoApproveSites=autoApproveList,
            custodialSubType=custodialType,
            nonCustodialSubType=nonCustodialType,
            custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
            nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
            priority=subscriptionPriority,
            deleteFromSource=kwargs.get("DeleteFromSource", False),
        )

        # Block closing information
        blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
        blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
        blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
        blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
        helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)

        helper.setMemoryAndCores(kwargs.get("Memory"), kwargs.get("Multicore"))
        helper.setDashboardActivity(kwargs.get("Dashboard", ""))
        helper.setTaskProperties(kwargs)

        Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)

        # update AcquisitionEra in the Couch document (#4380)
        # request object returned above from Oracle doesn't have information Couch
        # database
        reqDetails = Utilities.requestDetails(request["RequestName"])
        couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
        couchDb.updateDocument(
            request["RequestName"],
            "ReqMgr",
            "updaterequest",
            fields={
                "AcquisitionEra": reqDetails["AcquisitionEra"],
                "ProcessingVersion": reqDetails["ProcessingVersion"],
                "CustodialSites": custodialList,
                "NonCustodialSites": nonCustodialList,
                "AutoApproveSubscriptionSites": autoApproveList,
                "SubscriptionPriority": subscriptionPriority,
                "CustodialSubType": custodialType,
                "NonCustodialSubType": nonCustodialType,
                "CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
                "NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
                "DeleteFromSource": kwargs.get("DeleteFromSource", False),
                "Teams": kwargs["Teams"],
                "OutputDatasets": outputDatasets,
                "SiteWhitelist": whiteList,
                "SiteBlacklist": blackList,
                "MergedLFNBase": kwargs["MergedLFNBase"],
                "UnmergedLFNBase": kwargs["UnmergedLFNBase"],
                "Dashboard": kwargs.get("Dashboard", ""),
                "TrustSitelists": kwargs.get("TrustSitelists", False),
                "TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
                "AllowOpportunistic": kwargs.get("AllowOpportunistic", False),
            },
            useBody=True,
        )

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def updateRequestStats(requestName, stats, couchURL, couchDBName):
    couchDB = Database(couchDBName, couchURL)
    return couchDB.updateDocument(requestName, 'ReqMgr', 'totalstats',
                                         fields=stats)

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
    """
    If clone is True, the function is called on a cloned request in which
    case no modification of the reqSchema shall happen and should be checked in
    as is.
    
    """
    try:
        request = buildWorkloadForRequest(typename = reqSchema["RequestType"], 
                                          schema = reqSchema)
    except WMSpecFactoryException as ex:
        logging.error(traceback.format_exc())
        raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
    
    helper = WMWorkloadHelper(request['WorkloadSpec'])
    
    # update request as well for wmstats update
    # there is a better way to do this (passing helper to request but make sure all the information is there) 
    request["Campaign"] = helper.getCampaign()

    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request['OutputDatasets']:
                request['OutputDatasets'].append(ds)
    #TODO: need to update output dataset by Task for task chain requests
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)            
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema['RequestType'])
    except CheckIn.RequestCheckInError as ex:
        raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
        
    # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
    # metadata above is what is saved into couch to represent a request document.
    # Number of request arguments on a corresponding couch document
    # is not set, has default null/None values, update those accordingly now.
    # It's a mess to have two mutually inconsistent database backends.
    # Not easy to handle this earlier since couch is stored first and
    # some parameters are worked out later when storing into Oracle.
    reqDetails = requestDetails(request["RequestName"])
    # couchdb request parameters which are null at the injection time and remain so
    paramsToUpdate = ["RequestStatus",
                      "RequestSizeFiles",
                      "AcquisitionEra",
                      "RequestWorkflow",
                      "RequestType",
                      "RequestStatus",
                      "RequestPriority",
                      "Requestor",
                      "Group",
                      "SizePerEvent",
                      "PrepID",
                      "RequestNumEvents",
                      "ProcessingString",
                      "ProcessingVersion",
                      ]
    
    couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) 
        
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400, "Creating request failed, could not update WMStats.")

    return request

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
    def assignWorkload(self, requestName, kwargs):
        """ Make all the necessary changes in the Workload to reflect the new assignment """
        request = GetRequest.getRequestByName(requestName)
        helper = Utilities.loadWorkload(request)

        # Validate the different parts of the processed dataset
        processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
        if kwargs.get("ProcessingString", None):
            processedDatasetParts.append("ProcessingString")
        for field in processedDatasetParts:
            if type(kwargs[field]) == dict:
                for value in kwargs[field].values():
                    self.validate(value, field)
            else:
                self.validate(kwargs[field], field)

        # Set white list and black list
        whiteList = kwargs.get("SiteWhitelist", [])
        blackList = kwargs.get("SiteBlacklist", [])
        helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
        # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
        helper.setProcessingVersion(kwargs["ProcessingVersion"])
        helper.setAcquisitionEra(kwargs["AcquisitionEra"])
        helper.setProcessingString(kwargs.get("ProcessingString", None))
        # FIXME not validated
        helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
        helper.setMergeParameters(
            int(kwargs.get("MinMergeSize", 2147483648)),
            int(kwargs.get("MaxMergeSize", 4294967296)),
            int(kwargs.get("MaxMergeEvents", 50000)),
        )
        helper.setupPerformanceMonitoring(
            int(kwargs.get("maxRSS", 2411724)),
            int(kwargs.get("maxVSize", 2411724)),
            int(kwargs.get("SoftTimeout", 129600)),
            int(kwargs.get("GracePeriod", 300)),
        )

        # Check whether we should check location for the data
        if "useSiteListAsLocation" in kwargs:
            helper.setLocationDataSourceFlag()

        # Set phedex subscription information
        custodialList = kwargs.get("CustodialSites", [])
        nonCustodialList = kwargs.get("NonCustodialSites", [])
        autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
        subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
        if subscriptionPriority not in ["Low", "Normal", "High"]:
            raise cherrypy.HTTPError(400, "Invalid subscription priority")
        subscriptionType = kwargs.get("CustodialSubType", "Move")
        if subscriptionType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid custodial subscription type")

        helper.setSubscriptionInformationWildCards(
            wildcardDict=self.wildcardSites,
            custodialSites=custodialList,
            nonCustodialSites=nonCustodialList,
            autoApproveSites=autoApproveList,
            custodialSubType=subscriptionType,
            priority=subscriptionPriority,
        )

        # Block closing information
        blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
        blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
        blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
        blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))

        helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)

        helper.setDashboardActivity(kwargs.get("dashboard", ""))
        Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)

        # update AcquisitionEra in the Couch document (#4380)
        # request object returned above from Oracle doesn't have information Couch
        # database
        reqDetails = Utilities.requestDetails(request["RequestName"])
        couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
        couchDb.updateDocument(
            request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]}
        )

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
                      "RequestWorkflow",
                      "RequestType",
                      "RequestStatus",
                      "RequestPriority",
                      "Requestor",
                      "Group",
                      "SizePerEvent",
                      "PrepID",
                      "RequestNumEvents",
                      ]
    
    couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields) 
        
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400, "Creating request failed, could not update WMStats.")

    return request
    
        
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
    """
    Handles the submission of requests.
    

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
        subscriptionType = kwargs.get("CustodialSubType", "Move")
        if subscriptionType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % subscriptionType)

        helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
                                                   custodialSites = custodialList,
                                                   nonCustodialSites = nonCustodialList,
                                                   autoApproveSites = autoApproveList,
                                                   custodialSubType = subscriptionType,
                                                   priority = subscriptionPriority)

        # Block closing information
        blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
        blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
        blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
        blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))

        helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
                                     blockCloseMaxEvents, blockCloseMaxSize)

        helper.setDashboardActivity(kwargs.get("dashboard", ""))
        Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
        
        # update AcquisitionEra in the Couch document (#4380)
        # request object returned above from Oracle doesn't have information Couch
        # database
        reqDetails = Utilities.requestDetails(request["RequestName"])
        couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
        couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
                               fields={"AcquisitionEra": reqDetails["AcquisitionEra"]})

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
def main():
    if len(sys.argv) < 2:
        print("Missing the connect Oracle TNS argument (user/[email protected]).")
        sys.exit(1)
    tns = sys.argv[1]
    
    print("Creating CouchDB database connection ...")
    couchdb = Database(couchdb_name, couch_url)
    print("Creating Oracle database connection ...")
    oradb = cx_Oracle.Connection(tns)
    
    num_couch_requests = get_couchdb_row_count(couchdb)
    print("Total CouchDB request documents in ReqMgr: %s" % num_couch_requests)
    num_oracle_requests = get_oracle_row_count(oradb, "reqmgr_request")                                                
    print("Total Oracle requests entries in ReqMgr: %s" % num_oracle_requests)
        
    if num_couch_requests != num_oracle_requests:
        print("Number of requests in Oracle, CouchDB don't agree, fix that first.")
        sys.exit(1)
    else:
        print("Database cross-check (Oracle request names vs CouchDB): DONE, THE SAME.")
        
    
    def get_couch_value(couch_req, mapping):
        try:
            c = couch_req[mapping["couch"]]
            couch_missing = False
        except KeyError:            
            # comparison will not happen due to missing flag, the value
            # will be stored in couch
            c = "N/A"
            couch_missing = False
        return str(c), couch_missing
    
    
    def check_oracle_worflow_value(oracle_value, mapping, req_name):
        # check Oracle WORKFLOW value
        if mapping["oracle"] == "WORKFLOW":
            # https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/linacre_2011A_442p2_DataReprocessingMuOnia_111119_005717/spec
            from_wf_url_req_name = oracle_value.rsplit('/', 2)[-2]
            if req_name != from_wf_url_req_name:
                print("Workflow URL mismatch: %s" % o)
                sys.exit(1) 


    counter = 0
    for oracle_req in get_oracle_data(oradb):
        req_name = oracle_req["REQUEST_NAME"]

        # FILTER
        # check only requests injected approx. after last deployment (a lot of
        # stuff should have already been fixed in ReqMgr)
        # _13041._*$ (ending of request name with date/time)
        #if not re.match(".*_1304[0-3][0-9]_.*$", req_name): # all April 2013
        #    continue
        
        counter += 1
        print("\n\n%s (%s)" % (req_name, counter))        
                
        couch_req = couchdb.document(req_name)
        couch_fields_to_correct = {}
        for mapping in MAPPING:
            if mapping["couch"] in COUCH_TO_IGNORE:
                continue
            o = str(oracle_req[mapping["oracle"]])
            c, couch_missing = get_couch_value(couch_req, mapping)
            check_oracle_worflow_value(o, mapping, req_name)
            
            # compare oracle and couch values
            # don't update value in couch if it exists and is non-empty
            if (couch_missing or o != c) and c in ('None', '0', '', "N/A"):
                print("%s %s != %s" % (mapping, o, c))
                # correct couch request by oracle value
                couch_fields_to_correct[mapping["couch"]] = o
        
        if couch_fields_to_correct:
            print("Couch corrected fields:")
            print(couch_fields_to_correct)
            if sys.argv[-1] == "-c":
                couchdb.updateDocument(req_name, "ReqMgr", "updaterequest",
                                       fields=couch_fields_to_correct, useBody=True)
                print("Couch updated")
        else:
            print("OK")
        
        # fields that should be removed from couch
        """

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]
        helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
                                                   custodialSites = custodialList,
                                                   nonCustodialSites = nonCustodialList,
                                                   autoApproveSites = autoApproveList,
                                                   custodialSubType = subscriptionType,
                                                   priority = subscriptionPriority)

        # Block closing information
        blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
        blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
        blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
        blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))

        helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
                                     blockCloseMaxEvents, blockCloseMaxSize)

        helper.setDashboardActivity(kwargs.get("dashboard", ""))
        Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
        
        # update AcquisitionEra in the Couch document (#4380)
        # request object returned above from Oracle doesn't have information Couch
        # database
        reqDetails = Utilities.requestDetails(request["RequestName"])
        couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
        couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
                               fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
                                       "Teams": JsonWrapper.JSONEncoder().encode(kwargs["Teams"]),
                                       "SiteWhitelist": JsonWrapper.JSONEncoder().encode(whiteList),
                                       "SiteBlacklist": JsonWrapper.JSONEncoder().encode(blackList)})

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import updateDocument [as 别名]

#.........这里部分代码省略.........
            whiteList = [whiteList]
        if type(blackList) != list:
            blackList = [blackList]
        helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
                                     wildcardDict = self.wildcardSites)
        res = set(whiteList) & set(blackList)
        if len(res):
            raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
        # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
        helper.setProcessingVersion(kwargs["ProcessingVersion"])
        helper.setAcquisitionEra(kwargs["AcquisitionEra"])
        helper.setProcessingString(kwargs.get("ProcessingString", None))
        
        # Now verify the output datasets
        outputDatasets = helper.listOutputDatasets()
        for dataset in outputDatasets:
            tokens = dataset.split("/")
            procds = tokens[2]
            try:
                WMCore.Lexicon.procdataset(procds)
            except AssertionError as ex:
                raise cherrypy.HTTPError(400, 
                            "Bad output dataset name, check the processed dataset.\n %s" % 
                            str(ex))

        #FIXME not validated
        helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
        helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
                                  int(kwargs.get("MaxMergeSize", 4294967296)),
                                  int(kwargs.get("MaxMergeEvents", 50000)))
        helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
                                          kwargs.get("MaxVSize", None),
                                          kwargs.get("SoftTimeout",None),
                                          kwargs.get("GracePeriod", None))        

        # Check whether we should check location for the data
        useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
        if useAAA:
            helper.setLocationDataSourceFlag(flag = useAAA)

        # Set phedex subscription information
        custodialList = kwargs.get("CustodialSites", [])
        nonCustodialList = kwargs.get("NonCustodialSites", [])
        autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
        for site in autoApproveList:
            if site.endswith('_MSS'):
                raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
        subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
        if subscriptionPriority not in ["Low", "Normal", "High"]:
            raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
        custodialType = kwargs.get("CustodialSubType", "Replica")
        if custodialType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
        nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
        if nonCustodialType not in ["Move", "Replica"]:
            raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)

        helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
                                                   custodialSites = custodialList,
                                                   nonCustodialSites = nonCustodialList,
                                                   autoApproveSites = autoApproveList,
                                                   custodialSubType = custodialType,
                                                   nonCustodialSubType = nonCustodialType,
                                                   priority = subscriptionPriority)

        # Block closing information
        blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
        blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
        blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
        blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))

        helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
                                     blockCloseMaxEvents, blockCloseMaxSize)

        helper.setDashboardActivity(kwargs.get("Dashboard", ""))
        # set Task properties if they are exist
        # TODO: need to define the task format (maybe kwargs["tasks"]?)
        helper.setTaskProperties(kwargs)
        
        Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
        
        # update AcquisitionEra in the Couch document (#4380)
        # request object returned above from Oracle doesn't have information Couch
        # database
        reqDetails = Utilities.requestDetails(request["RequestName"])
        couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
        couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
                               fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
                                       "ProcessingVersion": reqDetails["ProcessingVersion"],
                                       "CustodialSites": custodialList, 
                                       "NonCustodialSites": nonCustodialList, 
                                       "AutoApproveSubscriptionSites": autoApproveList,
                                       "SubscriptionPriority": subscriptionPriority,
                                       "CustodialSubType": custodialType,
                                       "NonCustodialSubType": nonCustodialType,
                                       "Teams": kwargs["Teams"],
                                       "OutputDatasets": outputDatasets, 
                                       "SiteWhitelist": whiteList,
                                       "SiteBlacklist": blackList},
                               useBody = True)