Python RequestContainer.setSubRequestStatus方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]

#.........这里部分代码省略.........
                          lfn = res['Value']
                          selectedLfns.append( lfn )
                          lfnPfnDict[lfn] = pfn
                          pfnSize[pfn] = pfnDict['Size']

                      res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns )
                      if not res['OK']:
                        subDirs = [currentDir]
                      else:
                        for lfn in res['Value']['Failed'].keys():
                          gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
                          pfn = lfnPfnDict[lfn]
                          fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
                          res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if res['OK']:
                            gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, fuck, bugger. Add the failover." )

                        for lfn, lfnDict in res['Value']['Successful'].items():
                          pfn = lfnPfnDict[lfn]
                          storageSize = pfnSize[pfn]
                          catalogSize = lfnDict['Size']
                          if int( catalogSize ) == int( storageSize ):
                            gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
                            gLogger.info( "Change the status in the LFC" )
                          elif int( storageSize ) == 0:
                            gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) )
                            fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, fuck, bugger. Add the failover." )
                          else:
                            gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
                            fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, fuck, bugger. Add the failover." )

                        res = self.ReplicaManager.getCatalogReplicas( selectedLfns )
                        if not res['OK']:
                          subDirs = [currentDir]
                        else:
                          for lfn in res['Value']['Failed'].keys():
                            gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
                            pfn = lfnPfnDict[lfn]
                            fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
                            res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                            if res['OK']:
                              gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                              gLogger.error( "Change the status in the LFC,ProcDB...." )
                            else:
                              gLogger.error( "Shit, fuck, bugger. Add the failover." )

                          for lfn, repDict in res['Value']['Successful'].items():
                            pfn = lfnPfnDict[lfn]
                            registeredPfns = repDict.values()
                            if not pfn in registeredPfns:
                              gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) )
                              fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                              res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                              if res['OK']:
                                gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                                gLogger.error( "Change the status in the LFC,ProcDB...." )
                              else:
                                gLogger.error( "Shit, fuck, bugger. Add the failover." )
                            else:
                              gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn )

                    oNamespaceBrowser.updateDirs( subDirs )
                  oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation )

        ################################################
        #  Determine whether there are any active files
        if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
          oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )

      ################################################
      #  If the sub-request is already in terminal state
      else:
        gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]

#.........这里部分代码省略.........
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
      if subRequestAttributes['Status'] == 'Waiting':
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a lfcvsse operation
        if operation == 'LFCvsSE':
          gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation )
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile['LFN']
              oNamespaceBrowser = NamespaceBrowser( lfn )

              # Loop over all the directories and sub-directories
              while ( oNamespaceBrowser.isActive() ):
                currentDir = oNamespaceBrowser.getActiveDir()
                gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir )
                res = self.ReplicaManager.getCatalogDirectoryContents( currentDir )
                if not res['OK']:
                  subDirs = [currentDir]
                elif res['Value']['Failed'].has_key( currentDir ):
                  subDirs = [currentDir]
                else:
                  subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                  files = res['Value']['Successful'][currentDir]['Files']

                  lfnSizeDict = {}
                  pfnLfnDict = {}
                  pfnStatusDict = {}
                  sePfnDict = {}
                  for lfn, lfnDict in files.items():
                    lfnSizeDict[lfn] = lfnDict['MetaData']['Size']
                    for se in lfnDict['Replicas'].keys():
                      pfn = lfnDict['Replicas'][se]['PFN']
                      status = lfnDict['Replicas'][se]['Status']
                      pfnStatusDict[pfn] = status
                      pfnLfnDict[pfn] = lfn
                      if not sePfnDict.has_key( se ):
                        sePfnDict[se] = []
                      sePfnDict[se].append( pfn )

                  for storageElementName, physicalFiles in sePfnDict.items():
                    gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName )
                    res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName )
                    if not res['OK']:
                      gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] )
                    else:
                      for pfn in res['Value']['Failed'].keys():
                        gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) )
                        lfn = pfnLfnDict[pfn]
                        fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]}
                        IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                        resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                        if resInsert['OK']:
                          gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
                          gLogger.error( "Change the status in the LFC,ProcDB...." )
                        else:
                          gLogger.error( "Shit, fuck, bugger. Add the failover." )
                      for pfn, pfnDict in res['Value']['Successful'].items():
                        lfn = pfnLfnDict[pfn]
                        catalogSize = int( lfnSizeDict[lfn] )
                        storageSize = int( pfnDict['Size'] )
                        if int( catalogSize ) == int( storageSize ):
                          gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
                          gLogger.info( "Change the status in the LFC" )
                        else:
                          gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
                          fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                          IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                          resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if resInsert['OK']:
                            gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, fuck, bugger. Add the failover." )
                oNamespaceBrowser.updateDirs( subDirs )
              oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation )

        ################################################
        #  Determine whether there are any active files
        if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
          oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )

      ################################################
      #  If the sub-request is already in terminal state
      else:
        gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]
  def execute_request( self ):
    """ Takes one DISET request and forward it to the destination service
    """
    gMonitor.addMark( "Iteration", 1 )
    if self.RequestDB:
      res = self.RequestDB.getRequest( 'diset' )
    else:
      res = self.RequestDBClient.getRequest( 'diset' )
    if not res['OK']:
      gLogger.error( "DISETForwardingAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "DISETForwardingAgent.execute: No requests to be executed found." )
      return S_OK()

    gMonitor.addMark( "Attempted", 1 )
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    try:
      jobID = int( res['Value']['JobID'] )
    except:
      jobID = 0
    gLogger.info( "DISETForwardingAgent.execute: Obtained request %s" % requestName )

    if self.RequestDB:
      result = self.RequestDB._getRequestAttribute( 'RequestID', requestName = requestName )
      if not result['OK']:
        return S_OK( 'Can not get the request execution order' )
      requestID = result['Value']
      result = self.RequestDB.getCurrentExecutionOrder( requestID )
    else:
      result = self.RequestDBClient.getCurrentExecutionOrder( requestName )
    if result['OK']:
      currentOrder = result['Value']
    else:
      return S_OK( 'Can not get the request execution order' )

    oRequest = RequestContainer( request = requestString )
    requestAttributes = oRequest.getRequestAttributes()['Value']

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'diset' )
    if not res['OK']:
      errStr = "DISETForwardingAgent.execute: Failed to obtain number of diset subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()

    gLogger.info( "DISETForwardingAgent.execute: Found %s sub requests for job %s" % ( res['Value'], jobID ) )
    ################################################
    # For all the sub-requests in the request
    modified = False
    for ind in range( res['Value'] ):
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'diset' )['Value']
      subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
      subStatus = subRequestAttributes['Status']
      gLogger.info( "DISETForwardingAgent.execute: Processing sub-request %s with execution order %d" % ( ind, subExecutionOrder ) )
      if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
        operation = subRequestAttributes['Operation']
        gLogger.info( "DISETForwardingAgent.execute: Attempting to forward %s type." % operation )
        rpcStubString = subRequestAttributes['Arguments']
        rpcStub, length = DEncode.decode( rpcStubString )
        res = executeRPCStub( rpcStub )
        if res['OK']:
          gLogger.info( "DISETForwardingAgent.execute: Successfully forwarded." )
          oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
          gMonitor.addMark( "Successful", 1 )
          modified = True
        elif res['Message'] == 'No Matching Job':
          gLogger.warn( "DISETForwardingAgent.execute: No corresponding job found. Setting to done." )
          oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
        else:
          gLogger.error( "DISETForwardingAgent.execute: Failed to forward request.", res['Message'] )
      else:
        gLogger.info( "DISETForwardingAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    if self.RequestDB:
      res = self.RequestDB.updateRequest( requestName, requestString )
    else:
      res = self.RequestDBClient.updateRequest( requestName, requestString )
    if res['OK']:
      gLogger.info( "DISETForwardingAgent.execute: Successfully updated request." )
    else:
      gLogger.error( "DISETForwardingAgent.execute: Failed to update request" )

    if modified and jobID:
      result = self.RequestDBClient.finalizeRequest( requestName, jobID )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]
  def executeRequest( self ):
    ################################################
    # Get a request from request DB
    res = self.RequestDBClient.getRequest( 'register' )
    if not res['OK']:
      gLogger.info( "RegistrationAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "RegistrationAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    try:
      jobID = int( res['Value']['JobID'] )
    except:
      jobID = 0
    gLogger.info( "RegistrationAgent.execute: Obtained request %s" % requestName )

    result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
    if result['OK']:
      currentOrder = result['Value']
    else:
      return S_OK( 'Can not get the request execution order' )

    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'register' )
    if not res['OK']:
      errStr = "RegistrationAgent.execute: Failed to obtain number of transfer subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "RegistrationAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    modified = False
    for ind in range( res['Value'] ):
      gLogger.info( "RegistrationAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'register' )['Value']
      subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
      subStatus = subRequestAttributes['Status']
      if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'register' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a register file operation
        if operation == 'registerFile':
          gLogger.info( "RegistrationAgent.execute: Attempting to execute %s sub-request." % operation )
          diracSE = str( subRequestAttributes['TargetSE'] )
          if diracSE == 'SE':
            # We do not care about SE, put any there
            diracSE = "CERN-FAILOVER"
          catalog = subRequestAttributes['Catalogue']
          if catalog == "None":
            catalog = ''
          subrequest_done = True
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile.get( 'LFN', '' )
              if lfn: lfn = str( lfn )
              physicalFile = subRequestFile.get( 'PFN', '' )
              if physicalFile: physicalFile = str( physicalFile )
              fileSize = subRequestFile.get( 'Size', 0 )
              if fileSize: fileSize = int( fileSize )
              fileGuid = subRequestFile.get( 'GUID', '' )
              if fileGuid: fileGuid = str( fileGuid )
              checksum = subRequestFile.get( 'Addler', '' )
              if checksum: checksum = str( checksum )
              if catalog == 'BookkeepingDB':
                diracSE = 'CERN-HIST'
              fileTuple = ( lfn, physicalFile, fileSize, diracSE, fileGuid, checksum )
              res = self.ReplicaManager.registerFile( fileTuple, catalog )
              print res
              if not res['OK']:
                self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
                errStr = "RegistrationAgent.execute: Completely failed to register file."
                gLogger.error( errStr, res['Message'] )
                subrequest_done = False
              elif lfn in res['Value']['Failed'].keys():
                self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
                errStr = "RegistrationAgent.execute: Completely failed to register file."
                gLogger.error( errStr, res['Value']['Failed'][lfn] )
                subrequest_done = False
              else:
                self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
                oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                modified = True
            else:
              gLogger.info( "RegistrationAgent.execute: File already completed." )
          if subrequest_done:
            oRequest.setSubRequestStatus( ind, 'register', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.error( "RegistrationAgent.execute: Operation not supported.", operation )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]

#.........这里部分代码省略.........
        metadata = res['Value']['Successful']
        if not metadata.keys():
          gLogger.error( "ReplicationScheduler._execute: Failed to get metadata for all files." )
          continue

        ######################################################################################
        #
        # For each LFN determine the replication tree
        #

        for lfn in sortList( metadata.keys() ):
          fileSize = metadata[lfn]['Size']
          lfnReps = replicas[lfn]
          fileID = filesDict[lfn]

          targets = []
          for targetSE in targetSEs:
            if targetSE in lfnReps.keys():
              gLogger.debug( "ReplicationScheduler.execute: %s already present at %s." % ( lfn, targetSE ) )
            else:
              targets.append( targetSE )
          if not targets:
            gLogger.info( "ReplicationScheduler.execute: %s present at all targets." % lfn )
            oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
            continue
          if not lfnReps:
            gLogger.error( "ReplicationScheduler.execute: The file has no replicas.", lfn )
            continue
          res = self.strategyHandler.determineReplicationTree( sourceSE, targets, lfnReps, fileSize, strategy = reqRepStrategy )
          if not res['OK']:
            gLogger.error( "ReplicationScheduler.execute: Failed to determine replication tree.", res['Message'] )
            continue
          tree = res['Value']

          ######################################################################################
          #
          # For each item in the replication tree obtain the source and target SURLS
          #

          for channelID, dict in tree.items():
            gLogger.info( "ReplicationScheduler.execute: processing for channel %d %s" % ( channelID, str( dict ) ) )
            hopSourceSE = dict['SourceSE']
            hopDestSE = dict['DestSE']
            hopAncestor = dict['Ancestor']

            # Get the sourceSURL
            if hopAncestor:
              status = 'Waiting%s' % ( hopAncestor )
              res = self.obtainLFNSURL( hopSourceSE, lfn )
              if not res['OK']:
                errStr = res['Message']
                gLogger.error( errStr )
                return S_ERROR( errStr )
              sourceSURL = res['Value']
            else:
              status = 'Waiting'
              res = self.resolvePFNSURL( hopSourceSE, lfnReps[hopSourceSE] )
              if not res['OK']:
                sourceSURL = lfnReps[hopSourceSE]
              else:
                sourceSURL = res['Value']

            # Get the targetSURL
            res = self.obtainLFNSURL( hopDestSE, lfn )
            if not res['OK']:
              errStr = res['Message']
              gLogger.error( errStr )
              return S_ERROR( errStr )
            targetSURL = res['Value']

            ######################################################################################
            #
            # For each item in the replication tree add the file to the channel
            #
            res = self.TransferDB.addFileToChannel( channelID, fileID, hopSourceSE, sourceSURL, hopDestSE, targetSURL, fileSize, fileStatus = status )
            if not res['OK']:
              errStr = res['Message']
              gLogger.error( "ReplicationScheduler._execute: Failed to add File to Channel." , "%s %s" % ( fileID, channelID ) )
              return S_ERROR( errStr )
            res = self.TransferDB.addFileRegistration( channelID, fileID, lfn, targetSURL, hopDestSE )
            if not res['OK']:
              errStr = res['Message']
              gLogger.error( "ReplicationScheduler._execute: Failed to add File registration." , "%s %s" % ( fileID, channelID ) )
              result = self.TransferDB.removeFileFromChannel( channelID, fileID )
              if not result['OK']:
                errStr += result['Message']
                gLogger.error( "ReplicationScheduler._execute: Failed to remove File." , "%s %s" % ( fileID, channelID ) )
              return S_ERROR( errStr )
            oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Scheduled' )
          res = self.TransferDB.addReplicationTree( fileID, tree )

        if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
          oRequest.setSubRequestStatus( ind, 'transfer', 'Scheduled' )

      ################################################
      #  Generate the new request string after operation
      requestString = oRequest.toXML()['Value']
      res = self.RequestDB.updateRequest( requestName, requestString )
      if not res['OK']:
        gLogger.error( "ReplicationScheduler._execute: Failed to update request", "%s %s" % ( requestName, res['Message'] ) )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]

#.........这里部分代码省略.........
                  if errorMessage.find( 'Write access not permitted for this credential.' ) != -1:
                    if self.__getProxyAndRemoveReplica( diracSE, lfn ):
                      continue
                  if errorMessage.find( 'seconds timeout for "__gfal_wrapper" call' ) != -1:
                    self.timeOutCounter += 1
                  if not failed.has_key( lfn ):
                    failed[lfn] = {}
                  failed[lfn][diracSE] = res['Value']['Failed'][lfn]
              else:
                errMsg[diracSE] = res['Message']
                for lfn in lfns:
                  if not failed.has_key( lfn ):
                    failed[lfn] = {}
                  failed[lfn][diracSE] = 'Completely'
            # Now analyse the results
            failedLFNs = failed.keys()
            lfnsOK = [lfn for lfn in lfns if not lfn in failedLFNs]
            gMonitor.addMark( 'ReplicaRemovalDone', len( lfnsOK ) )
            for lfn in lfnsOK:
              gLogger.info( "RemovalAgent.execute: Successfully removed %s at %s" % ( lfn, str( diracSEs ) ) )
              res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
              if not res['OK']:
                gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
              modified = True
            if failed:
              gMonitor.addMark( 'PhysicalRemovalFail', len( failedLFNs ) )
              for lfn in failedLFNs:
                for diracSE in failed[lfn].keys():
                  if type( failed[lfn][diracSE] ) in StringTypes:
                    if re.search( 'no such file or directory', failed[lfn][diracSE].lower() ):
                      gLogger.info( "RemovalAgent.execute: File did not exist.", lfn )
                      res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
                      if not res['OK']:
                        gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
                      modified = True
                    else:
                      gLogger.info( "RemovalAgent.execute: Failed to remove file.", "%s at %s - %s" % ( lfn, diracSE, failed[lfn][diracSE] ) )
            if errMsg:
              for diracSE in errMsg.keys():
                errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
                gLogger.error( errStr, errMsg[diracSE] )

          ################################################
          #  If the sub-request is a request to the online system to retransfer
          elif operation == 'reTransfer':
            gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
            diracSE = subRequestAttributes['TargetSE']
            for subRequestFile in subRequestFiles:
              if subRequestFile['Status'] == 'Waiting':
                pfn = str( subRequestFile['PFN'] )
                lfn = str( subRequestFile['LFN'] )
                res = self.replicaManager.onlineRetransfer( diracSE, pfn )
                if res['OK']:
                  if res['Value']['Successful'].has_key( pfn ):
                    gLogger.info( "RemovalAgent.execute: Successfully requested retransfer of %s." % pfn )
                    result = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
                    if not result['OK']:
                      gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
                    modified = True
                  else:
                    errStr = "RemovalAgent.execute: Failed to request retransfer."
                    gLogger.error( errStr, "%s %s %s" % ( pfn, diracSE, res['Value']['Failed'][pfn] ) )
                else:
                  errStr = "RemovalAgent.execute: Completely failed to request retransfer."
                  gLogger.error( errStr, res['Message'] )
              else:
                gLogger.info( "RemovalAgent.execute: File already completed." )

          ################################################
          #  If the sub-request is none of the above types
          else:
            gLogger.error( "RemovalAgent.execute: Operation not supported.", operation )

          ################################################
          #  Determine whether there are any active files
          if oRequest.isSubRequestEmpty( ind, 'removal' )['Value']:
            oRequest.setSubRequestStatus( ind, 'removal', 'Done' )
            gMonitor.addMark( "Done", 1 )

        ################################################
        #  If the sub-request is already in terminal state
        else:
          gLogger.info( "RemovalAgent.execute:",
                        "Sub-request %s is status '%s' and not to be executed." %
                        ( ind, subRequestAttributes['Status'] ) )

      ################################################
      #  Generate the new request string after operation
      newrequestString = oRequest.toXML()['Value']
    except:
      # if something fails return the original request back to the server 
      res = self.requestDBClient.updateRequest( requestName, requestString, sourceServer )
      return S_OK()

    res = self.requestDBClient.updateRequest( requestName, newrequestString, sourceServer )

    if modified and jobID:
      result = self.finalizeRequest( requestName, jobID, sourceServer )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import setSubRequestStatus [as 别名]

#.........这里部分代码省略.........
                  gMonitor.addMark( "Replication failed", 1 )
                  errStr = "TransferAgent.execute: Failed to replicate and register file."
                  gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
                  
              else:
                gMonitor.addMark( "Replication failed", 1 )
                errStr = "TransferAgent.execute: Completely failed to replicate and register file."
                gLogger.error( errStr, res['Message'] )
                oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
                subRequestError = operation + " RM call failed"
            else:
              gLogger.info( "TransferAgent.execute: File already completed." )

        ################################################
        #  If the sub-request is a replicate operation
        elif operation == 'replicate':
          gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
          targetSE = subRequestAttributes['TargetSE']
          sourceSE = subRequestAttributes['SourceSE']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              gMonitor.addMark( "Replicate", 1 )
              lfn = subRequestFile['LFN']
              res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE = sourceSE )
              if res['OK']:
                if res['Value']['Successful'].has_key( lfn ):
                  gMonitor.addMark( "Replication successful", 1 )
                  gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
                  oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                  modified = True
                else:
                  gMonitor.addMark( "Replication failed", 1 )
                  errStr = "TransferAgent.execute: Failed to replicate file."
                  gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
                  subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE )
                  oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
              else:
                gMonitor.addMark( "Replication failed", 1 )
                errStr = "TransferAgent.execute: Completely failed to replicate file."
                gLogger.error( errStr, res['Message'] )
                subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE )
                oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Replicate RM call failed' )
            else:
              gLogger.info( "TransferAgent.execute: File already completed." )

        ################################################
        #  If the sub-request is a get operation
        elif operation == 'get':
          gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
          sourceSE = subRequestAttributes['TargetSE']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = str( subRequestFile['LFN'] )
              pfn = str( subRequestFile['PFN'] )
              got = False
              if sourceSE and pfn:
                res = self.ReplicaManager.getStorageFile( pfn, sourceSE )
                if res['Value']['Successful'].has_key( pfn ):
                  got = True
              else:
                res = self.ReplicaManager.getFile( lfn )
                if res['Value']['Successful'].has_key( lfn ):
                  got = False
              if got:
                gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn )
                oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                modified = True
              else:
                errStr = "TransferAgent.execute: Failed to get file."
                gLogger.error( errStr, lfn )
            else:
              gLogger.info( "TransferAgent.execute: File already completed." )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.error( "TransferAgent.execute: Operation not supported.", operation )

        if subRequestError:
          oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError )

        ################################################
        #  Determine whether there are any active files
        if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
          oRequest.setSubRequestStatus( ind, 'transfer', 'Done' )
          gMonitor.addMark( "Done", 1 )

      ################################################
      #  If the sub-request is already in terminal state
      else:
        gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, subRequestAttributes['Status'] ) )

    ################################################
    #  Generate the new request string after operation
    requestString = oRequest.toXML()['Value']
    res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )

    if modified and jobID:
      result = self.finalizeRequest( requestName, jobID, sourceServer )
    return S_OK()