Python HTTPRequests.post方法代码示例

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def mark_failed(ids, failures_reasons):
    """
    Mark the list of files as failed
    :param ids: list of Oracle file ids to update
    :param failures_reasons: list of strings with transfer failure messages
    :return: 0 success, 1 failure
    """
    try:
        oracleDB = HTTPRequests(rest_filetransfers,
                                proxy,
                                proxy)
        data = dict()
        data['asoworker'] = 'asoless'
        data['subresource'] = 'updateTransfers'
        data['list_of_ids'] = ids
        data['list_of_transfer_state'] = ["FAILED" for _ in ids]
        data['list_of_failure_reason'] = failures_reasons
        data['list_of_retry_value'] = [0 for _ in ids]

        oracleDB.post('/filetransfers',
                      data=encodeRequest(data))
        logging.debug("Marked failed %s", ids)
    except Exception:
        logging.exception("Error updating documents")
        return 1
    return 0

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def mark_transferred(ids):
    """
    Mark the list of files as tranferred
    :param ids: list of Oracle file ids to update
    :return: 0 success, 1 failure
    """
    try:
        oracleDB = HTTPRequests(rest_filetransfers,
                                proxy,
                                proxy)
        logging.debug("Marking done %s", ids)

        data = dict()
        data['asoworker'] = 'asoless'
        data['subresource'] = 'updateTransfers'
        data['list_of_ids'] = ids
        data['list_of_transfer_state'] = ["DONE" for _ in ids]

        oracleDB.post('/filetransfers',
                      data=encodeRequest(data))
        logging.debug("Marked good %s", ids)
    except Exception:
        logging.exception("Error updating documents")
        return 1
    return 0

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
 def uploadWarning(self, warning, userProxy, taskname):
     try:
         userServer = HTTPRequests(self.server["host"], userProxy, userProxy, retry=2)
         configreq = {"subresource": "addwarning", "workflow": taskname, "warning": b64encode(warning)}
         userServer.post(self.restURInoAPI + "/task", data=urllib.urlencode(configreq))
     except HTTPException as hte:
         self.logger.error(hte.headers)
         self.logger.warning("Cannot add a warning to REST interface. Warning message: %s" % warning)

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
 def deleteWarnings(self, userProxy, taskname):
     userServer = HTTPRequests(self.server['host'], userProxy, userProxy, retry=2,
                               logger = self.logger)
     configreq = {'subresource': 'deletewarnings',
                     'workflow': taskname}
     try:
         userServer.post(self.restURInoAPI + '/task', data = urllib.urlencode(configreq))
     except HTTPException as hte:
         self.logger.error("Error deleting warnings: %s", str(hte))
         self.logger.warning("Can not delete warnings from REST interface.")

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
    def execute(self, *args, **kwargs):
        wmwork = Workflow(name=kwargs['task']['tm_taskname'])

        wmsubs = Subscription(fileset=args[0], workflow=wmwork,
                               split_algo=kwargs['task']['tm_split_algo'],
                               type=self.jobtypeMapper[kwargs['task']['tm_job_type']])
        splitter = SplitterFactory()
        jobfactory = splitter(subscription=wmsubs)
        splitparam = kwargs['task']['tm_split_args']
        splitparam['algorithm'] = kwargs['task']['tm_split_algo']
        if kwargs['task']['tm_job_type'] == 'Analysis':
            if kwargs['task']['tm_split_algo'] == 'FileBased':
                splitparam['total_files'] = kwargs['task']['tm_totalunits']
            elif kwargs['task']['tm_split_algo'] == 'LumiBased':
                splitparam['total_lumis'] = kwargs['task']['tm_totalunits']
            elif kwargs['task']['tm_split_algo'] == 'EventAwareLumiBased':
                splitparam['total_events'] = kwargs['task']['tm_totalunits']
        elif kwargs['task']['tm_job_type'] == 'PrivateMC':
            if 'tm_events_per_lumi' in kwargs['task'] and kwargs['task']['tm_events_per_lumi']:
                splitparam['events_per_lumi'] = kwargs['task']['tm_events_per_lumi']
            if 'tm_generator' in kwargs['task'] and kwargs['task']['tm_generator'] == 'lhe':
                splitparam['lheInputFiles'] = True
        splitparam['applyLumiCorrection'] = True
        factory = jobfactory(**splitparam)
        numJobs = sum([len(jobgroup.getJobs()) for jobgroup in factory])
        maxJobs = getattr(self.config.TaskWorker, 'maxJobsPerTask', 10000)
        if numJobs == 0:
            msg  = "The CRAB3 server backend could not submit any job to the Grid scheduler:"
            msg += " Splitting task %s" % (kwargs['task']['tm_taskname'])
            if kwargs['task']['tm_input_dataset']:
                msg += " on dataset %s" % (kwargs['task']['tm_input_dataset'])
            msg += " with %s method does not generate any job" % (kwargs['task']['tm_split_algo'])
            raise TaskWorkerException(msg)
        elif numJobs > maxJobs:
            raise TaskWorkerException("The splitting on your task generated %s jobs. The maximum number of jobs in each task is %s" %
                                        (numJobs, maxJobs))
        #printing duplicated lumis if any
        lumiChecker = getattr(jobfactory, 'lumiChecker', None)
        if lumiChecker and lumiChecker.splitLumiFiles:
            self.logger.warning("The input dataset contains the following duplicated lumis %s" % lumiChecker.splitLumiFiles.keys())
            #TODO use self.uploadWarning
            try:
                userServer = HTTPRequests(self.server['host'], kwargs['task']['user_proxy'], kwargs['task']['user_proxy'], retry = 2,
                                          logger = self.logger)
                configreq = {'subresource': 'addwarning',
                             'workflow': kwargs['task']['tm_taskname'],
                             'warning': b64encode('The CRAB3 server backend detected lumis split across files in the input dataset.'
                                        ' Will apply the necessary corrections in the splitting algorithms. You can ignore this message.')}
                userServer.post(self.restURInoAPI + '/task', data = urllib.urlencode(configreq))
            except HTTPException as hte:
                self.logger.error(hte.headers)
                self.logger.warning("Cannot add warning to REST after finding duplicates")

        return Result(task = kwargs['task'], result = factory)

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
 def uploadWarning(self, warning, userProxy, taskname):
     try:
         userServer = HTTPRequests(self.server['host'], userProxy, userProxy, retry=2,
                                   logger = self.logger)
         configreq = {'subresource': 'addwarning',
                      'workflow': taskname,
                      'warning': b64encode(warning)}
         userServer.post(self.restURInoAPI + '/task', data = urllib.urlencode(configreq))
     except HTTPException as hte:
         self.logger.error(hte.headers)
         self.logger.warning("Cannot add a warning to REST interface. Warning message: %s" % warning)

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
 def sendScheddToREST(self, task, schedd):
     """ Try to set the schedd to the oracle database in the REST interface
         Raises TaskWorkerException in case of failure
     """
     task['tm_schedd'] = schedd
     userServer = HTTPRequests(self.server['host'], task['user_proxy'], task['user_proxy'], retry=20, logger=self.logger)
     configreq = {'workflow':task['tm_taskname'], 'subresource':'updateschedd',
         'scheddname':schedd}
     try:
         userServer.post(self.restURInoAPI + '/task', data=urllib.urlencode(configreq))
     except HTTPException as hte:
         msg = "Unable to contact cmsweb and update scheduler on which task will be submitted. Error msg: %s" % hte.headers
         self.logger.warning(msg)
         time.sleep(20)
         raise TaskWorkerException(msg) #we already tried 20 times, give up

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
    def uploadWarning(self, warning, userProxy, taskname):
        if not self.server: # When testing, the server can be None
            self.logger.warning(warning)
            return

        truncWarning = truncateError(warning)
        userServer = HTTPRequests(self.server['host'], userProxy, userProxy, retry=2,
                                  logger = self.logger)
        configreq = {'subresource': 'addwarning',
                        'workflow': taskname,
                         'warning': b64encode(truncWarning)}
        try:
            userServer.post(self.restURInoAPI + '/task', data = urllib.urlencode(configreq))
        except HTTPException as hte:
            self.logger.error("Error uploading warning: %s", str(hte))
            self.logger.warning("Cannot add a warning to REST interface. Warning message: %s", warning)

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def updatewebdir(ad):
    data = {'subresource' : 'addwebdir'}
    host = ad['CRAB_RestHost']
    uri = ad['CRAB_RestURInoAPI'] + '/task'
    data['workflow'] = ad['CRAB_ReqName']
    data['webdirurl'] = ad['CRAB_UserWebDir']
    cert = ad['X509UserProxy']
    try:
        from RESTInteractions import HTTPRequests
        from httplib import HTTPException
        import urllib
        server = HTTPRequests(host, cert, cert)
        server.post(uri, data = urllib.urlencode(data))
        return 0
    except:
        print traceback.format_exc()
        return 1

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def updateWebDir(ad):
    """
    Need a doc string here.
    """
    data = {'subresource' : 'addwebdir'}
    host = ad['CRAB_RestHost']
    uri = ad['CRAB_RestURInoAPI'] + '/task'
    data['workflow'] = ad['CRAB_ReqName']
    data['webdirurl'] = ad['CRAB_UserWebDir']
    cert = ad['X509UserProxy']
    try:
        server = HTTPRequests(host, cert, cert)
        server.post(uri, data = urllib.urlencode(data))
        return 0
    except HTTPException as hte:
        printLog(traceback.format_exc())
        printLog(hte.headers)
        printLog(hte.result)
        return 1

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def updateWebDir(ad):
    """
    Need a doc string here.
    """
    data = {"subresource": "addwebdir"}
    host = ad["CRAB_RestHost"]
    uri = ad["CRAB_RestURInoAPI"] + "/task"
    data["workflow"] = ad["CRAB_ReqName"]
    data["webdirurl"] = ad["CRAB_UserWebDir"]
    cert = ad["X509UserProxy"]
    try:
        server = HTTPRequests(host, cert, cert)
        server.post(uri, data=urllib.urlencode(data))
        return 0
    except HTTPException as hte:
        printLog(traceback.format_exc())
        printLog(hte.headers)
        printLog(hte.result)
        return 1

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
    def __call__(self):
        ## retrieving output files location from the server
        server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)

        self.logger.debug('Requesting resubmission for failed jobs in task %s' % self.cachedinfo['RequestName'] )
        #inputdict = { "TaskResubmit": "Analysis", "ForceResubmit" : force }
        dictresult, status, reason = server.post(self.uri, data = urlencode({ 'workflow' : self.cachedinfo['RequestName']}) + \
                                                    self.sitewhitelist + self.siteblacklist + '&' + urlencode(self.jobids))
        self.logger.debug("Result: %s" % dictresult)

        if status != 200:
            msg = "Problem retrieving resubmitting the task to the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputdict), str(dictresult), str(reason))
            raise RESTCommunicationException(msg)

        self.logger.info("Resubmit request successfully sent")
        if dictresult['result'][0]['result'] != 'ok':
            self.logger.info(dictresult['result'][0]['result'])

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
    def __call__(self):
        server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)

        self.logger.debug('Requesting resubmission for failed jobs in task %s' % self.cachedinfo['RequestName'] )
        configreq = { 'workflow' : self.cachedinfo['RequestName']}

        for attr in ['maxmemory', 'maxjobruntime', 'numcores', 'priority']:
            val = getattr(self, attr, None)
            if val:
                configreq[attr] = val

        dictresult, status, reason = server.post(self.uri, data = urlencode({ 'workflow' : self.cachedinfo['RequestName']}) + \
                                                    self.sitewhitelist + self.siteblacklist + '&' + urlencode(self.jobids))
        self.logger.debug("Result: %s" % dictresult)

        if status != 200:
            msg = "Problem retrieving resubmitting the task to the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputdict), str(dictresult), str(reason))
            raise RESTCommunicationException(msg)

        self.logger.info("Resubmit request successfully sent")
        if dictresult['result'][0]['result'] != 'ok':
            self.logger.info(dictresult['result'][0]['result'])

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
def processWorker(inputs, results, resthost, resturi, procnum):
    """Wait for an reference to appear in the input queue, call the referenced object
       and write the output in the output queue.

       :arg Queue inputs: the queue where the inputs are shared by the master
       :arg Queue results: the queue where this method writes the output
       :return: default returning zero, but not really needed."""
    logger = setProcessLogger(str(procnum))
    logger.info("Process %s is starting. PID %s", procnum, os.getpid())
    procName = "Process-%s" % procnum
    while True:
        try:
            workid, work, task, inputargs = inputs.get()
        except (EOFError, IOError):
            crashMessage = "Hit EOF/IO in getting new work\n"
            crashMessage += "Assuming this is a graceful break attempt.\n"
            logger.error(crashMessage)
            break
        if work == 'STOP':
            break

        outputs = None
        t0 = time.time()
        logger.debug("%s: Starting %s on %s" %(procName, str(work), task['tm_taskname']))
        try:
            msg = None
            outputs = work(resthost, resturi, WORKER_CONFIG, task, procnum, inputargs)
        except WorkerHandlerException as we:
            outputs = Result(task=task, err=str(we))
            msg = str(we)
        except Exception as exc:
            outputs = Result(task=task, err=str(exc))
            msg = "%s: I just had a failure for %s" % (procName, str(exc))
            msg += "\n\tworkid=" + str(workid)
            msg += "\n\ttask=" + str(task['tm_taskname'])
            msg += "\n" + str(traceback.format_exc())
        finally:
            if msg:
                try:
                    logger.info("Uploading error message to REST: %s" % msg)
                    server = HTTPRequests(resthost, WORKER_CONFIG.TaskWorker.cmscert, WORKER_CONFIG.TaskWorker.cmskey, retry = 2)
                    truncMsg = truncateError(msg)
                    configreq = {  'workflow': task['tm_taskname'],
                                   'status': "FAILED",
                                   'subresource': 'failure',
                                   #limit the message to 7500 chars, which means no more than 10000 once encoded. That's the limit in the REST
                                   'failure': b64encode(truncMsg)}

                    server.post(resturi, data = urllib.urlencode(configreq))
                    logger.info("Error message successfully uploaded to the REST")
                except HTTPException as hte:
                    logger.warning("Cannot upload failure message to the REST for workflow %s. HTTP headers follows:" % task['tm_taskname'])
                    logger.error(hte.headers)
                except Exception as exc:
                    logger.warning("Cannot upload failure message to the REST for workflow %s.\nReason: %s" % (task['tm_taskname'], exc))
                    logger.exception('Traceback follows:')
        t1 = time.time()
        logger.debug("%s: ...work on %s completed in %d seconds: %s" % (procName, task['tm_taskname'], t1-t0, outputs))

        results.put({
                     'workid': workid,
                     'out' : outputs
                    })
    logger.debug("Slave %s exiting." % procnum)
    return 0

# 需要导入模块: from RESTInteractions import HTTPRequests [as 别名]
# 或者: from RESTInteractions.HTTPRequests import post [as 别名]
class RetryManagerDaemon(BaseDaemon):
    """
    _RetryManagerPoller_

    Polls for Files in CoolOff State and attempts to retry them
    based on the requirements in the selected plugin
    """
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseDaemon.__init__(self, config, 'RetryManager')

        if self.config.isOracle:
            try:
                self.oracleDB = HTTPRequests(self.config.oracleDB,
                                             self.config.opsProxy,
                                             self.config.opsProxy)
            except:
                self.logger.exception('Failed to connect to Oracle')
        else:
            try:
                server = CouchServer(dburl=self.config.couch_instance,
                                     ckey=self.config.opsProxy,
                                     cert=self.config.opsProxy)
                self.db = server.connectDatabase(self.config.files_database)
            except Exception as e:
                self.logger.exception('A problem occured when connecting to couchDB: %s' % e)
                raise
            self.logger.debug('Connected to files DB')

            # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.retryAlgoDir, namespace=self.config.retryAlgoDir)
        try:
            self.plugin = self.factory.loadObject(self.config.algoName, self.config,
                                                  getFromCache=False, listFlag=True)
        except Exception as ex:
            msg = "Error loading plugin %s on path %s\n" % (self.config.algoName,
                                                            self.config.retryAlgoDir)
            msg += str(ex)
            self.logger.error(msg)
            raise RetryManagerException(msg)
        self.cooloffTime = self.config.cooloffTime

    def terminate(self, params):
        """
        Run one more time through, then terminate

        """
        logging.debug("Terminating. doing one more pass before we die")
        self.algorithm(params)

    def algorithm(self, parameters=None):
        """
        Performs the doRetries method, loading the appropriate
        plugin for each job and handling it.
        """
        logging.debug("Running retryManager algorithm")
        if self.config.isOracle:
            fileDoc = dict()
            fileDoc['asoworker'] = self.config.asoworker
            fileDoc['subresource'] = 'retryTransfers'
            fileDoc['time_to'] = self.cooloffTime
            self.logger.debug('fileDoc: %s' % fileDoc)
            try:
                results = self.oracleDB.post(self.config.oracleFileTrans,
                                             data=encodeRequest(fileDoc))
            except Exception:
                self.logger.exception("Failed to get retry transfers in oracleDB: %s")
                return
            logging.info("Retried files in cooloff: %s,\n now getting transfers to kill" % str(results))

            fileDoc = dict()
            fileDoc['asoworker'] = self.config.asoworker
            fileDoc['subresource'] = 'getTransfersToKill'
            fileDoc['grouping'] = 0
            try:
                results = self.oracleDB.get(self.config.oracleFileTrans,
                                            data=encodeRequest(fileDoc))
                result = oracleOutputMapping(results)
            except Exception as ex:
                self.logger.error("Failed to get killed transfers \
                                   from oracleDB: %s" % ex)
                return

            usersToKill = list(set([(x['username'], x['user_group'], x['user_role']) for x in result]))

            self.logger.debug("Users with transfers to kill: %s" % usersToKill)
            transfers = Queue()

            for i in range(self.config.kill_threads):
                worker = Thread(target=self.killThread, args=(i, transfers,))
                worker.setDaemon(True)
                worker.start()

            for user in usersToKill:
                user_trans = [x for x in result if (x['username'], x['user_group'], x['user_role']) == user]
                self.logger.info("Inserting %s transfers of user %s in the killing queue" % (len(user_trans), user))
                transfers.put(user_trans)

#.........这里部分代码省略.........