Python ReqClient.getRequestFileStatus方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestFileStatus [as 别名]

#.........这里部分代码省略.........
    """ Submit requests one by one
    """
    submitted = 0
    failed = 0
    startTime = time.time()
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self.log.error( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self.log.info( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self.log.info( 'submitTasks: Failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )

  def updateTransformationReservedTasks( self, taskDicts ):
    taskNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      transID = taskDict['TransformationID']
      taskID = taskDict['TaskID']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      res = self.requestClient.getRequestInfo( taskName )
      if res['OK']:
        taskNameIDs[taskName] = res['Value'][0]
      elif re.search( "Failed to retrieve RequestID for Request", res['Message'] ):
        noTasks.append( taskName )
      else:
        self.log.warn( "Failed to get requestID for request", res['Message'] )
    return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':taskNameIDs} )

  def getSubmittedTaskStatus( self, taskDicts ):
    updateDict = {}
    for taskDict in taskDicts:
      transID = taskDict['TransformationID']
      taskID = taskDict['TaskID']
      oldStatus = taskDict['ExternalStatus']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      res = self.requestClient.getRequestStatus( taskName )
      newStatus = ''
      if res['OK']:
        # FIXME: for compatibility between old and new RMS
        try:
          # old
          newStatus = res['Value']['RequestStatus']
        except TypeError:
          # new
          newStatus = res['Value']
      elif re.search( "Failed to retrieve RequestID for Request", res['Message'] ):
        newStatus = 'Failed'
      else:
        self.log.info( "getSubmittedTaskStatus: Failed to get requestID for request", res['Message'] )
      if newStatus and ( newStatus != oldStatus ):
        if newStatus not in updateDict:
          updateDict[newStatus] = []
        updateDict[newStatus].append( taskID )
    return S_OK( updateDict )

  def getSubmittedFileStatus( self, fileDicts ):
    taskFiles = {}
    for fileDict in fileDicts:
      transID = fileDict['TransformationID']
      taskID = fileDict['TaskID']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      if taskName not in taskFiles:
        taskFiles[taskName] = {}
      taskFiles[taskName][fileDict['LFN']] = fileDict['Status']

    updateDict = {}
    for taskName in sorted( taskFiles ):
      lfnDict = taskFiles[taskName]
      res = self.requestClient.getRequestFileStatus( taskName, lfnDict.keys() )
      if not res['OK']:
        self.log.warn( "getSubmittedFileStatus: Failed to get files status for request", res['Message'] )
        continue
      for lfn, newStatus in res['Value'].items():
        if newStatus == lfnDict[lfn]:
          pass
        elif newStatus == 'Done':
          updateDict[lfn] = 'Processed'
        elif newStatus == 'Failed':
          updateDict[lfn] = 'Problematic'
    return S_OK( updateDict )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestFileStatus [as 别名]

#.........这里部分代码省略.........
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self._logError( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )

  def updateTransformationReservedTasks( self, taskDicts ):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] )

      reqID = taskDict['ExternalID']

      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append( requestName )
    return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} )


  def getSubmittedTaskStatus( self, taskDicts ):
    updateDict = {}

    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']

      newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] )
      if not newStatus['OK']:
        log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn
        log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] )
      else:
        newStatus = newStatus['Value']
        if newStatus != oldStatus:
          updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] )
    return S_OK( updateDict )

  def getSubmittedFileStatus( self, fileDicts ):
    taskFiles = {}
    submittedTasks = {}
    externalIds = {}
    # Don't try and get status of not submitted tasks!
    for fileDict in fileDicts:
      submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) )
    for transID in submittedTasks:
      res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} )
      if not res['OK']:
        return res
      for taskDict in res['Value']:
        taskID = taskDict['TaskID']
        externalIds[taskID] = taskDict['ExternalID']
        if taskDict['ExternalStatus'] == 'Created':
          submittedTasks[transID].remove( taskID )

    for fileDict in fileDicts:
      transID = fileDict['TransformationID']
      taskID = int( fileDict['TaskID'] )
      if taskID in submittedTasks[transID]:
        requestID = externalIds[taskID]
        taskFiles.setdefault( requestID, {} )[fileDict['LFN']] = fileDict['Status']

    updateDict = {}
    for requestID in sorted( taskFiles ):
      lfnDict = taskFiles[requestID]
      statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys() )
      if not statusDict['OK']:
        log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn
        log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] )
        continue

      statusDict = statusDict['Value']
      for lfn, newStatus in statusDict.items():
        if newStatus == lfnDict[lfn]:
          pass
        elif newStatus == 'Done':
          updateDict[lfn] = 'Processed'
        elif newStatus == 'Failed':
          updateDict[lfn] = 'Problematic'
    return S_OK( updateDict )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestFileStatus [as 别名]

#.........这里部分代码省略.........
        submitted += 1
      else:
        self._logError("Failed to submit task to RMS", res['Message'], transID=transID)
        task['Success'] = False
        failed += 1
    if submitted:
      self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime),
                    transID=transID, method=method)
    if failed:
      self._logWarn('Failed to submit %d tasks to RMS.' % (failed),
                    transID=transID, method=method)
    return S_OK(taskDict)

  def submitTaskToExternal(self, oRequest):
    """
    Submits a request to RMS
    """
    if isinstance(oRequest, self.requestClass):
      return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2)
    return S_ERROR("Request should be a Request object")

  def updateTransformationReservedTasks(self, taskDicts):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID'])
      reqID = taskDict['ExternalID']
      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append(requestName)
    return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})

  def getSubmittedTaskStatus(self, taskDicts):
    """
    Check if tasks changed status, and return a list of tasks per new status
    """
    updateDict = {}
    badRequestID = 0
    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']
      # ExternalID is normally a string
      if taskDict['ExternalID'] and int(taskDict['ExternalID']):
        newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID'])
        if not newStatus['OK']:
          log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn
          log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'],
              transID=taskDict['TransformationID'])
        else:
          newStatus = newStatus['Value']
          # We don't care updating the tasks to Assigned while the request is being processed
          if newStatus != oldStatus and newStatus != 'Assigned':
            updateDict.setdefault(newStatus, []).append(taskDict['TaskID'])
      else:
        badRequestID += 1
    if badRequestID:
      self._logWarn("%d requests have identifier 0" % badRequestID)
    return S_OK(updateDict)

  def getSubmittedFileStatus(self, fileDicts):
    """
    Check if transformation files changed status, and return a list of taskIDs per new status
    """
    # Don't try and get status of not submitted tasks!
    transID = None
    taskFiles = {}
    for fileDict in fileDicts:
      # There is only one transformation involved, get however the transID in the loop
      transID = fileDict['TransformationID']
      taskID = int(fileDict['TaskID'])
      taskFiles.setdefault(taskID, []).append(fileDict['LFN'])
    # Should not happen, but just in case there are no files, return
    if transID is None:
      return S_OK({})

    res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()})
    if not res['OK']:
      return res
    requestFiles = {}
    for taskDict in res['Value']:
      taskID = taskDict['TaskID']
      externalID = taskDict['ExternalID']
      # Only consider tasks that are submitted, ExternalID is a string
      if taskDict['ExternalStatus'] != 'Created' and externalID and int(externalID):
        requestFiles[externalID] = taskFiles[taskID]

    updateDict = {}
    for requestID, lfnList in requestFiles.iteritems():
      statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList)
      if not statusDict['OK']:
        log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn
        log("Failed to get files status for request", statusDict['Message'],
            transID=transID, method='getSubmittedFileStatus')
      else:
        for lfn, newStatus in statusDict['Value'].iteritems():
          if newStatus == 'Done':
            updateDict[lfn] = 'Processed'
          elif newStatus == 'Failed':
            updateDict[lfn] = 'Problematic'
    return S_OK(updateDict)