Python ReqClient.getRequestStatus方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]
    if not request:
      gLogger.error( "no such request %s" % requestName )
      continue
    if status and request.Status != status:
      if not warningPrinted:
        gLogger.always( "Some requests are not in status %s" % status )
        warningPrinted = True
      continue

    if all or recoverableRequest( request ):
      okRequests.append( requestName )
      if reset:
        gLogger.always( '============ Request %s =============' % requestName )
        ret = reqClient.resetFailedRequest( requestName, all = all )
        if not ret['OK']:
          gLogger.error( "Error resetting request %s" % requestName, ret['Message'] )
      else:
        if len( requests ) > 1:
          gLogger.always( '\n===================================' )
        dbStatus = reqClient.getRequestStatus( requestName ).get( 'Value', 'Unknown' )

        printRequest( request, status = dbStatus, full = full, verbose = verbose, terse = terse )
  if status and okRequests:
    from DIRAC.Core.Utilities.List import breakListIntoChunks
    gLogger.always( '\nList of %d selected requests:' % len( okRequests ) )
    for reqs in breakListIntoChunks( okRequests, 100 ):
      gLogger.always( ','.join( reqs ) )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]

#.........这里部分代码省略.........
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self._logError( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )

  def updateTransformationReservedTasks( self, taskDicts ):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] )

      reqID = taskDict['ExternalID']

      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append( requestName )
    return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} )


  def getSubmittedTaskStatus( self, taskDicts ):
    updateDict = {}

    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']

      newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] )
      if not newStatus['OK']:
        log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn
        log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] )
      else:
        newStatus = newStatus['Value']
        if newStatus != oldStatus:
          updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] )
    return S_OK( updateDict )

  def getSubmittedFileStatus( self, fileDicts ):
    taskFiles = {}
    submittedTasks = {}
    externalIds = {}
    # Don't try and get status of not submitted tasks!
    for fileDict in fileDicts:
      submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) )
    for transID in submittedTasks:
      res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} )
      if not res['OK']:
        return res
      for taskDict in res['Value']:
        taskID = taskDict['TaskID']
        externalIds[taskID] = taskDict['ExternalID']
        if taskDict['ExternalStatus'] == 'Created':
          submittedTasks[transID].remove( taskID )

    for fileDict in fileDicts:
      transID = fileDict['TransformationID']
      taskID = int( fileDict['TaskID'] )
      if taskID in submittedTasks[transID]:
        requestID = externalIds[taskID]
        taskFiles.setdefault( requestID, {} )[fileDict['LFN']] = fileDict['Status']

    updateDict = {}
    for requestID in sorted( taskFiles ):
      lfnDict = taskFiles[requestID]
      statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys() )
      if not statusDict['OK']:
        log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn
        log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] )
        continue

      statusDict = statusDict['Value']
      for lfn, newStatus in statusDict.items():
        if newStatus == lfnDict[lfn]:
          pass
        elif newStatus == 'Done':
          updateDict[lfn] = 'Processed'
        elif newStatus == 'Failed':
          updateDict[lfn] = 'Problematic'
    return S_OK( updateDict )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]

#.........这里部分代码省略.........
    """ Submit requests one by one
    """
    submitted = 0
    failed = 0
    startTime = time.time()
    for taskID in sorted( taskDict ):
      if not taskDict[taskID]['TaskObject']:
        taskDict[taskID]['Success'] = False
        failed += 1
        continue
      res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
      if res['OK']:
        taskDict[taskID]['ExternalID'] = res['Value']
        taskDict[taskID]['Success'] = True
        submitted += 1
      else:
        self.log.error( "Failed to submit task to RMS", res['Message'] )
        taskDict[taskID]['Success'] = False
        failed += 1
    self.log.info( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
    if failed:
      self.log.info( 'submitTasks: Failed to submit %d tasks to RMS.' % ( failed ) )
    return S_OK( taskDict )

  def submitTaskToExternal( self, oRequest ):
    """ Submits a request using ReqClient
    """
    if isinstance( oRequest, self.requestClass ):
      return self.requestClient.putRequest( oRequest )
    else:
      return S_ERROR( "Request should be a Request object" )

  def updateTransformationReservedTasks( self, taskDicts ):
    taskNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      transID = taskDict['TransformationID']
      taskID = taskDict['TaskID']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      res = self.requestClient.getRequestInfo( taskName )
      if res['OK']:
        taskNameIDs[taskName] = res['Value'][0]
      elif re.search( "Failed to retrieve RequestID for Request", res['Message'] ):
        noTasks.append( taskName )
      else:
        self.log.warn( "Failed to get requestID for request", res['Message'] )
    return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':taskNameIDs} )

  def getSubmittedTaskStatus( self, taskDicts ):
    updateDict = {}
    for taskDict in taskDicts:
      transID = taskDict['TransformationID']
      taskID = taskDict['TaskID']
      oldStatus = taskDict['ExternalStatus']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      res = self.requestClient.getRequestStatus( taskName )
      newStatus = ''
      if res['OK']:
        # FIXME: for compatibility between old and new RMS
        try:
          # old
          newStatus = res['Value']['RequestStatus']
        except TypeError:
          # new
          newStatus = res['Value']
      elif re.search( "Failed to retrieve RequestID for Request", res['Message'] ):
        newStatus = 'Failed'
      else:
        self.log.info( "getSubmittedTaskStatus: Failed to get requestID for request", res['Message'] )
      if newStatus and ( newStatus != oldStatus ):
        if newStatus not in updateDict:
          updateDict[newStatus] = []
        updateDict[newStatus].append( taskID )
    return S_OK( updateDict )

  def getSubmittedFileStatus( self, fileDicts ):
    taskFiles = {}
    for fileDict in fileDicts:
      transID = fileDict['TransformationID']
      taskID = fileDict['TaskID']
      taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
      if taskName not in taskFiles:
        taskFiles[taskName] = {}
      taskFiles[taskName][fileDict['LFN']] = fileDict['Status']

    updateDict = {}
    for taskName in sorted( taskFiles ):
      lfnDict = taskFiles[taskName]
      res = self.requestClient.getRequestFileStatus( taskName, lfnDict.keys() )
      if not res['OK']:
        self.log.warn( "getSubmittedFileStatus: Failed to get files status for request", res['Message'] )
        continue
      for lfn, newStatus in res['Value'].items():
        if newStatus == lfnDict[lfn]:
          pass
        elif newStatus == 'Done':
          updateDict[lfn] = 'Processed'
        elif newStatus == 'Failed':
          updateDict[lfn] = 'Problematic'
    return S_OK( updateDict )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]

#.........这里部分代码省略.........
        submitted += 1
      else:
        self._logError("Failed to submit task to RMS", res['Message'], transID=transID)
        task['Success'] = False
        failed += 1
    if submitted:
      self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime),
                    transID=transID, method=method)
    if failed:
      self._logWarn('Failed to submit %d tasks to RMS.' % (failed),
                    transID=transID, method=method)
    return S_OK(taskDict)

  def submitTaskToExternal(self, oRequest):
    """
    Submits a request to RMS
    """
    if isinstance(oRequest, self.requestClass):
      return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2)
    return S_ERROR("Request should be a Request object")

  def updateTransformationReservedTasks(self, taskDicts):
    requestNameIDs = {}
    noTasks = []
    for taskDict in taskDicts:
      requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID'])
      reqID = taskDict['ExternalID']
      if reqID:
        requestNameIDs[requestName] = reqID
      else:
        noTasks.append(requestName)
    return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})

  def getSubmittedTaskStatus(self, taskDicts):
    """
    Check if tasks changed status, and return a list of tasks per new status
    """
    updateDict = {}
    badRequestID = 0
    for taskDict in taskDicts:
      oldStatus = taskDict['ExternalStatus']
      # ExternalID is normally a string
      if taskDict['ExternalID'] and int(taskDict['ExternalID']):
        newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID'])
        if not newStatus['OK']:
          log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn
          log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'],
              transID=taskDict['TransformationID'])
        else:
          newStatus = newStatus['Value']
          # We don't care updating the tasks to Assigned while the request is being processed
          if newStatus != oldStatus and newStatus != 'Assigned':
            updateDict.setdefault(newStatus, []).append(taskDict['TaskID'])
      else:
        badRequestID += 1
    if badRequestID:
      self._logWarn("%d requests have identifier 0" % badRequestID)
    return S_OK(updateDict)

  def getSubmittedFileStatus(self, fileDicts):
    """
    Check if transformation files changed status, and return a list of taskIDs per new status
    """
    # Don't try and get status of not submitted tasks!
    transID = None
    taskFiles = {}
    for fileDict in fileDicts:
      # There is only one transformation involved, get however the transID in the loop
      transID = fileDict['TransformationID']
      taskID = int(fileDict['TaskID'])
      taskFiles.setdefault(taskID, []).append(fileDict['LFN'])
    # Should not happen, but just in case there are no files, return
    if transID is None:
      return S_OK({})

    res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()})
    if not res['OK']:
      return res
    requestFiles = {}
    for taskDict in res['Value']:
      taskID = taskDict['TaskID']
      externalID = taskDict['ExternalID']
      # Only consider tasks that are submitted, ExternalID is a string
      if taskDict['ExternalStatus'] != 'Created' and externalID and int(externalID):
        requestFiles[externalID] = taskFiles[taskID]

    updateDict = {}
    for requestID, lfnList in requestFiles.iteritems():
      statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList)
      if not statusDict['OK']:
        log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn
        log("Failed to get files status for request", statusDict['Message'],
            transID=transID, method='getSubmittedFileStatus')
      else:
        for lfn, newStatus in statusDict['Value'].iteritems():
          if newStatus == 'Done':
            updateDict[lfn] = 'Processed'
          elif newStatus == 'Failed':
            updateDict[lfn] = 'Problematic'
    return S_OK(updateDict)

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]
  subRequestIndex = oRequest.initiateSubRequest( requestType )['Value']
  attributeDict = {'Operation':requestOperation, 'TargetSE':targetSE}
  oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict )
  files = []
  for lfn in lfnList:
    files.append( {'LFN':lfn} )
  oRequest.setSubRequestFiles( subRequestIndex, requestType, files )
  requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
  oRequest.setRequestAttributes( {'RequestName':requestName} )

  DIRAC.gLogger.info( oRequest.toXML()['Value'] )

  result = reqClient.setRequest( requestName, oRequest.toXML()['Value'] )
  if result['OK']:
    print 'Submitted Request:', result['Value']
  else:
    print 'Failed to submit Request', result['Message']
  if monitor:
    requestID = result['Value']
    while True:
      result = reqClient.getRequestStatus( requestID )
      if not result['OK']:
        Script.gLogger.error( result['Message'] )
        break
      Script.gLogger.notice( result['Value']['RequestStatus'] )
      if result['Value']['RequestStatus'] == 'Done':
        break
      import time
      time.sleep( 10 )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient import getRequestStatus [as 别名]

#.........这里部分代码省略.........

    counter = 0
    for lfnChunk in breakListIntoChunks(list(lfnCache), 200):
      counter += 200
      if counter % 1000 == 0:
        self.log.notice('Getting FileInfo: %d/%d: %3.1fs' % (counter, len(jobs), float(time.time() - self.startTime)))
      while True:
        try:
          reps = self.fcClient.exists(lfnChunk)
          if not reps['OK']:
            self.log.error('Failed to check file existence, try again...', reps['Message'])
            raise RuntimeError('Try again')
          statuses = reps['Value']
          lfnExistence.update(statuses['Successful'])
          break
        except RuntimeError:  # try again
          pass

    return lfnExistence

  def setPendingRequests(self, jobs):
    """Loop over all the jobs and get requests, if any."""
    for jobChunk in breakListIntoChunks(jobs.values(), 1000):
      jobIDs = [job.jobID for job in jobChunk]
      while True:
        result = self.reqClient.readRequestsForJobs(jobIDs)
        if result['OK']:
          break
        self.log.error('Failed to read requests', result['Message'])
        # repeat
      for jobID in result['Value']['Successful']:
        request = result['Value']['Successful'][jobID]
        requestID = request.RequestID
        dbStatus = self.reqClient.getRequestStatus(requestID).get('Value', 'Unknown')
        for job in jobChunk:
          if job.jobID == jobID:
            job.pendingRequest = dbStatus not in ('Done', 'Canceled')
            self.log.notice('Found %s request for job %d' % ('pending' if job.pendingRequest else 'finished', jobID))
            break

  def checkAllJobs(self, jobs, tInfo, tasksDict=None, lfnTaskDict=None):
    """run over all jobs and do checks"""
    fileJobDict = defaultdict(list)
    counter = 0
    nJobs = len(jobs)
    self.setPendingRequests(jobs)
    lfnExistence = self.getLFNStatus(jobs)
    self.log.notice('Running over all the jobs')
    for counter, job in enumerate(jobs.values()):
      if counter % self.printEveryNJobs == 0:
        self.log.notice('%d/%d: %3.1fs' % (counter, nJobs, float(time.time() - self.startTime)))
      while True:
        try:
          if job.pendingRequest:
            self.log.warn('Job has Pending requests:\n%s' % job)
            break
          job.checkFileExistence(lfnExistence)
          if tasksDict and lfnTaskDict:
            try:
              job.getTaskInfo(tasksDict, lfnTaskDict)
            except TaskInfoException as e:
              self.log.error(" Skip Task, due to TaskInfoException: %s" % e )
              if job.inputFile is None and not job.tType.startswith( "MCGeneration" ):
                self.__failJobHard(job, tInfo)
              break
            fileJobDict[job.inputFile].append( job.jobID )