Python ReqClient.putRequest方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
  def putRequest( self, userName, userDN, userGroup, sourceSE, targetSE1, targetSE2 ):
    """ test case for user """

    req = self.buildRequest( userName, userGroup, sourceSE, targetSE1, targetSE2 )

    req.RequestName = "test%s-%s" % ( userName, userGroup )
    req.OwnerDN = userDN
    req.OwnerGroup = userGroup

    gLogger.always( "putRequest: request '%s'" % req.RequestName )
    for op in req:
      gLogger.always( "putRequest: => %s %s %s" % ( op.Order, op.Type, op.TargetSE ) )
      for f in op:
        gLogger.always( "putRequest: ===> file %s" % f.LFN )

    reqClient = ReqClient()

    delete = reqClient.deleteRequest( req.RequestName )
    if not delete["OK"]:
      gLogger.error( "putRequest: %s" % delete["Message"] )
      return delete
    put = reqClient.putRequest( req )
    if not put["OK"]:
      gLogger.error( "putRequest: %s" % put["Message"] )
    return put

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
def myRequest():
  """Create a request and put it to the db"""

  request = Request()
  request.RequestName = 'myAwesomeRemovalRequest.xml'
  request.JobID = 0
  request.SourceComponent = "myScript"

  remove = Operation()
  remove.Type = "RemoveFile"

  lfn = "/ilc/user/s/sailer/test.txt"
  rmFile = File()
  rmFile.LFN = lfn
  remove.addFile( rmFile )

  request.addOperation( remove )
  isValid = RequestValidator().validate( request )
  if not isValid['OK']:
    raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
  else:
    print("It is a GOGOGO")
    requestClient = ReqClient()
    result = requestClient.putRequest( request )
    print(result)

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
  def commitRequest( self ):
    """ Send request to the Request Management Service
    """
    if self.request.isEmpty():
      return S_OK()

    isValid = RequestValidator().validate( self.request )
    if not isValid["OK"]:
      return S_ERROR( "Failover request is not valid: %s" % isValid["Message"] )
    else:
      requestClient = ReqClient()
      result = requestClient.putRequest( self.request )
      return result

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
    print "Can't get file metadata: %s" % res['Message']
    DIRAC.exit( 1 )
  if res['Value']['Failed']:
    print "Could not get the file metadata of the following, so skipping them:"
    for fFile in res['Value']['Failed']:
      print fFile

  lfnMetadata = res['Value']['Successful']

  for lfn in lfnMetadata:
    rarFile = File()
    rarFile.LFN = lfn
    rarFile.Size = lfnMetadata[lfn]['Size']
    rarFile.Checksum = lfnMetadata[lfn]['Checksum']
    rarFile.GUID = lfnMetadata[lfn]['GUID']
    rarFile.ChecksumType = 'ADLER32'
    replicateAndRegister.addFile( rarFile )

  oRequest.addOperation( replicateAndRegister )
  isValid = RequestValidator().validate( oRequest )
  if not isValid['OK']:
    print "Request is not valid: ", isValid['Message']
    DIRAC.exit( 1 )

  result = reqClient.putRequest( oRequest )
  if result['OK']:
    print "Request %d submitted successfully" % result['Value']
  else:
    print "Failed to submit Request: ", result['Message']

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
    for lfn in lfnChunk:
      metaDict = metaDatas["Successful"][lfn]
      opFile = File()
      opFile.LFN = lfn
      opFile.Size = metaDict["Size"]

      if "Checksum" in metaDict:
        # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
        opFile.Checksum = metaDict["Checksum"]
        opFile.ChecksumType = "ADLER32"
      replicateAndRegister.addFile( opFile )

    request.addOperation( replicateAndRegister )

    putRequest = reqClient.putRequest( request )
    if not putRequest["OK"]:
      gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
      error = -1
      continue
    requestIDs.append( str( putRequest["Value"] ) )
    if not multiRequests:
      gLogger.always( "Request '%s' has been put to ReqDB for execution." % request.RequestName )

  if multiRequests:
    gLogger.always( "%d requests have been put to ReqDB for execution, with name %s_<num>" % ( count, requestName ) )
  if requestIDs:
    gLogger.always( "RequestID(s): %s" % " ".join( requestIDs ) )
  gLogger.always( "You can monitor requests' status using command: 'dirac-rms-show-request <requestName/ID>'" )
  DIRAC.exit( error )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]

#.........这里部分代码省略.........
      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )

  def submitTransformationTasks( self, taskDict ):
    """ Submit requests one by one
    """
    submitted = 0
    failed = 0
    startTime = time.time()
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self._logError( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )

  def updateTransformationReservedTasks( self, taskDicts ):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] )

      reqID = taskDict['ExternalID']

      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append( requestName )
    return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} )


  def getSubmittedTaskStatus( self, taskDicts ):
    updateDict = {}

    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']

      newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] )
      if not newStatus['OK']:
        log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn
        log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] )
      else:
        newStatus = newStatus['Value']
        if newStatus != oldStatus:
          updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]

#.........这里部分代码省略.........
    return res

  def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
    """ Prepare the new jobs that have to be submitted

        :param maxFilesPerJob: maximum number of files assigned to a job
        :param maxAttemptsPerFile: maximum number of retry after an fts failure

        :return list of jobs
    """
    raise NotImplementedError("You should not be using the base class")

  def _updateRmsOperationStatus(self):
    """ Update the status of the Files in the rms operation
          :return: S_OK with a dict:
                        * request: rms Request object
                        * operation: rms Operation object
                        * ftsFilesByTarget: dict {SE: [ftsFiles that were successful]}
    """

    log = self._log.getSubLogger("_updateRmsOperationStatus/%s/%s" %
                                 (getattr(self, 'operationID'), self.rmsReqID), child=True)

    res = self.reqClient.getRequest(self.rmsReqID)
    if not res['OK']:
      return res

    request = res['Value']

    res = request.getWaiting()

    if not res["OK"]:
      log.error("Unable to find 'Scheduled' operation in request")
      res = self.reqClient.putRequest(request, useFailoverProxy=False, retryMainService=3)
      if not res['OK']:
        log.error("Could not put back the request !", res['Message'])
      return S_ERROR("Could not find scheduled operation")

    operation = res['Value']

    # We index the files of the operation by their IDs
    rmsFileIDs = {}

    for opFile in operation:
      rmsFileIDs[opFile.FileID] = opFile

    # Files that failed to transfer
    defunctRmsFileIDs = set()

    # { SE : [FTS3Files] }
    ftsFilesByTarget = {}
    for ftsFile in self.ftsFiles:

      if ftsFile.status == 'Defunct':
        log.info(
            "File failed to transfer, setting it to failed in RMS", "%s %s" %
            (ftsFile.lfn, ftsFile.targetSE))
        defunctRmsFileIDs.add(ftsFile.rmsFileID)
        continue

      if ftsFile.status == 'Canceled':
        log.info(
            "File canceled, setting it Failed in RMS", "%s %s" %
            (ftsFile.lfn, ftsFile.targetSE))
        defunctRmsFileIDs.add(ftsFile.rmsFileID)
        continue

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
class RequestTasks( TaskBase ):

  def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, ):
    """ c'tor

        the requestClass is by default Request.
        If extensions want to use an extended type, they can pass it as a parameter.
        This is the same behavior as WorfkloTasks and jobClass
    """

    if not logger:
      logger = gLogger.getSubLogger( 'RequestTasks' )

    super( RequestTasks, self ).__init__( transClient, logger )

    if not requestClient:
      self.requestClient = ReqClient()
    else:
      self.requestClient = requestClient

    if not requestClass:
      self.requestClass = Request
    else:
      self.requestClass = requestClass

  def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
    """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
    """
    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split( ';' )
      except AttributeError:
        pass

    for taskID in sorted( taskDict ):
      paramDict = taskDict[taskID]
      if paramDict['InputData']:
        transID = paramDict['TransformationID']

        oRequest = Request()
        transfer = Operation()
        transfer.Type = requestOperation
        transfer.TargetSE = paramDict['TargetSE']

        if type( paramDict['InputData'] ) == type( [] ):
          files = paramDict['InputData']
        elif type( paramDict['InputData'] ) == type( '' ):
          files = paramDict['InputData'].split( ';' )
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile( trFile )

        oRequest.addOperation( transfer )
        oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
        oRequest.OwnerDN = owner
        oRequest.OwnerGroup = ownerGroup

      isValid = gRequestValidator.validate( oRequest )
      if not isValid['OK']:
        return isValid

      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )

  def submitTransformationTasks( self, taskDict ):
    """ Submit requests one by one
    """
    submitted = 0
    failed = 0
    startTime = time.time()
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self.log.error( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self.log.info( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self.log.info( 'submitTasks: Failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]

#.........这里部分代码省略.........
    submitted = 0
    failed = 0
    startTime = time.time()
    method = 'submitTransformationTasks'
    for task in taskDict.itervalues():
      # transID is the same for all tasks, so pick it up every time here
      transID = task['TransformationID']
      if not task['TaskObject']:
        task['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal(task['TaskObject'])
      if res['OK']:
        task['ExternalID'] = res['Value']
        task['Success'] = True
        submitted += 1
      else:
        self._logError("Failed to submit task to RMS", res['Message'], transID=transID)
        task['Success'] = False
        failed += 1
    if submitted:
      self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime),
                    transID=transID, method=method)
    if failed:
      self._logWarn('Failed to submit %d tasks to RMS.' % (failed),
                    transID=transID, method=method)
    return S_OK(taskDict)

  def submitTaskToExternal(self, oRequest):
    """
    Submits a request to RMS
    """
    if isinstance(oRequest, self.requestClass):
      return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2)
    return S_ERROR("Request should be a Request object")

  def updateTransformationReservedTasks(self, taskDicts):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID'])
      reqID = taskDict['ExternalID']
      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append(requestName)
    return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})

  def getSubmittedTaskStatus(self, taskDicts):
    """
    Check if tasks changed status, and return a list of tasks per new status
    """
    updateDict = {}
    badRequestID = 0
    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']
      # ExternalID is normally a string
      if taskDict['ExternalID'] and int(taskDict['ExternalID']):
        newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID'])
        if not newStatus['OK']:
          log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn
          log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'],
              transID=taskDict['TransformationID'])
        else:
          newStatus = newStatus['Value']
          # We don't care updating the tasks to Assigned while the request is being processed

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]

#.........这里部分代码省略.........
    for key, status in (("Att", "Attempted"), ("OK", "Successful"), ("Fail", "Failed")):
      gMonitor.registerActivity(
          "%s%s" %
          (pluginName, key), "%s operations %s" %
          (pluginName, status), "RequestExecutingAgent", "Operations/min", gMonitor.OP_SUM)
    # # return an instance
    return pluginClassObj

  def getHandler(self, operation):
    """ return instance of a handler for a given operation type on demand
        all created handlers are kept in self.handlers dict for further use

    :param ~Operation.Operation operation: Operation instance
    """
    if operation.Type not in self.handlersDict:
      return S_ERROR("handler for operation '%s' not set" % operation.Type)
    handler = self.handlers.get(operation.Type, None)
    if not handler:
      try:
        handlerCls = self.loadHandler(self.handlersDict[operation.Type])
        self.handlers[operation.Type] = handlerCls(
            csPath="%s/OperationHandlers/%s" % (self.csPath, operation.Type))
        handler = self.handlers[operation.Type]
      except (ImportError, TypeError) as error:
        self.log.exception("getHandler: %s" % str(error), lException=error)
        return S_ERROR(str(error))
    # # set operation for this handler
    handler.setOperation(operation)
    # # and return
    return S_OK(handler)

  def updateRequest(self):
    """ put back request to the RequestDB """
    updateRequest = self.requestClient.putRequest(
        self.request, useFailoverProxy=False, retryMainService=2)
    if not updateRequest["OK"]:
      self.log.error(updateRequest["Message"])
    return updateRequest

  def __call__(self):
    """ request processing """

    self.log.debug("about to execute request")
    gMonitor.addMark("RequestAtt", 1)

    # # setup proxy for request owner
    setupProxy = self.setupProxy()
    if not setupProxy["OK"]:
      self.request.Error = setupProxy["Message"]
      if 'has no proxy registered' in setupProxy["Message"]:
        self.log.error('Request set to Failed:', setupProxy["Message"])
        # If user is no longer registered, fail the request
        for operation in self.request:
          for opFile in operation:
            opFile.Status = 'Failed'
          operation.Status = 'Failed'
      else:
        self.log.error(setupProxy["Message"])
      return S_OK(self.request)
    shifter = setupProxy["Value"]["Shifter"]
    proxyFile = setupProxy["Value"]["ProxyFile"]

    error = None
    while self.request.Status == "Waiting":

      # # get waiting operation

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import putRequest [as 别名]
    for copies in copies_at:
      transfer = Operation()
      transfer.Type = "ReplicateAndRegister"
      transfer.TargetSE = copies
      trFile = File()
      trFile.LFN = lfnpath
      trFile.GUID = ""
      transfer.addFile(trFile)
      request.addOperation(transfer)

    res = RequestValidator().validate(request)
    if not res['OK']:
      return res

    if copies_at:
      res = requestClient.putRequest(request)
      if not res['OK']:
        gLogger.error('Could not set replication request', res['Message'])
      return S_OK('Application uploaded')
  return S_OK()

def fullCopy(srcdir, dstdir, item):
  """ Copy the item from srcdir to dstdir, creates missing directories if needed
  """
  item = item.rstrip().lstrip().lstrip("./").rstrip("/")
  srcdir = srcdir.rstrip("/")
  dstdir = dstdir.rstrip("/")
  if not re.match(r"(.*)[a-zA-Z0-9]+(.*)", item):#we want to have explicit elements
    gLogger.error("You try to get all files, that cannot happen")
    return S_OK()
  src = os.path.join(srcdir, item)