Python RequestContainer.getSubRequestFiles方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]
  def execute( self ):

    res = self.RequestDBClient.getRequest( 'integrity' )
    if not res['OK']:
      gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    gLogger.info( "LFCvsSEAgent.execute: Obtained request %s" % requestName )
    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'integrity' )
    if not res['OK']:
      errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "LFCvsSEAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    for ind in range( res['Value'] ):
      gLogger.info( "LFCvsSEAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
      if subRequestAttributes['Status'] == 'Waiting':
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a lfcvsse operation
        if operation == 'LFCvsSE':
          gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation )
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile['LFN']
              oNamespaceBrowser = NamespaceBrowser( lfn )

              # Loop over all the directories and sub-directories
              while ( oNamespaceBrowser.isActive() ):
                currentDir = oNamespaceBrowser.getActiveDir()
                gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir )
                res = self.ReplicaManager.getCatalogDirectoryContents( currentDir )
                if not res['OK']:
                  subDirs = [currentDir]
                elif res['Value']['Failed'].has_key( currentDir ):
                  subDirs = [currentDir]
                else:
                  subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                  files = res['Value']['Successful'][currentDir]['Files']

                  lfnSizeDict = {}
                  pfnLfnDict = {}
                  pfnStatusDict = {}
                  sePfnDict = {}
                  for lfn, lfnDict in files.items():
                    lfnSizeDict[lfn] = lfnDict['MetaData']['Size']
                    for se in lfnDict['Replicas'].keys():
                      pfn = lfnDict['Replicas'][se]['PFN']
                      status = lfnDict['Replicas'][se]['Status']
                      pfnStatusDict[pfn] = status
                      pfnLfnDict[pfn] = lfn
                      if not sePfnDict.has_key( se ):
                        sePfnDict[se] = []
                      sePfnDict[se].append( pfn )

                  for storageElementName, physicalFiles in sePfnDict.items():
                    gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName )
                    res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName )
                    if not res['OK']:
                      gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] )
                    else:
                      for pfn in res['Value']['Failed'].keys():
                        gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) )
                        lfn = pfnLfnDict[pfn]
                        fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]}
                        IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                        resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                        if resInsert['OK']:
                          gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
                          gLogger.error( "Change the status in the LFC,ProcDB...." )
                        else:
                          gLogger.error( "Shit, fuck, bugger. Add the failover." )
                      for pfn, pfnDict in res['Value']['Successful'].items():
                        lfn = pfnLfnDict[pfn]
                        catalogSize = int( lfnSizeDict[lfn] )
                        storageSize = int( pfnDict['Size'] )
                        if int( catalogSize ) == int( storageSize ):
                          gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
                          gLogger.info( "Change the status in the LFC" )
                        else:
                          gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
                          fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
                          IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
                          resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if resInsert['OK']:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]

#.........这里部分代码省略.........
    """

    ##############################################################
    # here comes the processing
    ##############################################################
    res = self.requestObj.getNumSubRequests( self.__requestType )
    if not res["OK"]:
      errMsg = "handleRequest: failed to obtain number of '%s' subrequests." % self.__requestType
      self.error( errMsg, res["Message"] )
      return S_ERROR( res["Message"] )

    ## for gMonitor
    self.addMark( "Execute", 1 )
    ## process sub requests
    for index in range( res["Value"] ):
      self.info( "handleRequest: processing subrequest %s." % str(index) )
      subRequestAttrs = self.requestObj.getSubRequestAttributes( index, self.__requestType )["Value"]
      if subRequestAttrs["ExecutionOrder"]:
        subExecutionOrder = int( subRequestAttrs["ExecutionOrder"] )
      else:
        subExecutionOrder = 0
      subRequestStatus = subRequestAttrs["Status"]
      if subRequestStatus != "Waiting":
        self.info( "handleRequest: subrequest %s has status '%s' and is not to be executed." % ( str(index), 
                                                                                                 subRequestStatus ) )
        continue
      
      if subExecutionOrder <= self.executionOrder:
        operation = subRequestAttrs["Operation"]
        if operation not in self.operationDispatcher():
          self.error( "handleRequest: '%s' operation not supported" % operation )
        else:
          self.info( "handleRequest: will execute %s '%s' subrequest" % ( str(index), operation ) )

          ## get files
          subRequestFiles = self.requestObj.getSubRequestFiles( index, self.__requestType )["Value"]          
          ## execute operation action
          ret = self.operationDispatcher()[operation].__call__( index,
                                                                self.requestObj,
                                                                subRequestAttrs,
                                                                subRequestFiles )
          ################################################
          ## error in operation action?
          if not ret["OK"]:
            self.error( "handleRequest: error when handling subrequest %s: %s" % ( str(index),  ret["Message"] ) )
            self.requestObj.setSubRequestAttributeValue( index, self.__requestType, "Error", ret["Message"] )
          else:
            ## update ref to requestObj
            self.requestObj = ret["Value"]
            ## check if subrequest status == Done, disable finalisation if not
            subRequestDone = self.requestObj.isSubRequestDone( index, self.__requestType )
            if not subRequestDone["OK"]:
              self.error( "handleRequest: unable to determine subrequest status: %s" % subRequestDone["Message"] )
            else:
              if not subRequestDone["Value"]:
                self.warn("handleRequest: subrequest %s is not done yet" % str(index) )

    ################################################
    #  Generate the new request string after operation
    newRequestString = self.requestObj.toXML()['Value']
    update = self.putBackRequest( self.requestName, newRequestString )
    if not update["OK"]:
      self.error( "handleRequest: error when updating request: %s" % update["Message"] )
      return update

    ## get request status                
    if self.jobID:
      requestStatus = self.requestClient().getRequestStatus( self.requestName )
      if not requestStatus["OK"]:
        return requestStatus
      requestStatus = requestStatus["Value"]
      ## finalize request if jobID is present and request status = 'Done'
      self.info("handleRequest: request status is %s" % requestStatus )
      
      if ( requestStatus["RequestStatus"] == "Done" ) and ( requestStatus["SubRequestStatus"] not in ( "Waiting", "Assigned" ) ):
        self.debug("handleRequest: request is going to be finalised")
        finalize = self.requestClient().finalizeRequest( self.requestName, self.jobID )
        if not finalize["OK"]:
          self.error("handleRequest: error in request finalization: %s" % finalize["Message"] )
          return finalize
        self.info("handleRequest: request is finalised")
      ## for gMonitor
      self.addMark( "Done", 1 )

    ## should return S_OK with monitor dict
    return S_OK( { "monitor" : self.monitor() } )
 
  def putBackRequest( self, requestName, requestString ):
    """ put request back

    :param self: self reference
    :param str requestName: request name
    :param str requestString: XML-serilised request
    :param str sourceServer: request server URL
    """
    update = self.requestClient().updateRequest( requestName, requestString )
    if not update["OK"]:
      self.error( "putBackRequest: error when updating request: %s" % update["Message"] )
      return update
    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]
  def execute( self ):

    IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )

    res = self.RequestDBClient.getRequest( 'integrity' )
    if not res['OK']:
      gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['requestString']
    requestName = res['Value']['requestName']
    sourceServer = res['Value']['Server']
    gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName )
    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'integrity' )
    if not res['OK']:
      errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    for ind in range( res['Value'] ):
      gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
      if subRequestAttributes['Status'] == 'Waiting':
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a lfcvsse operation
        if operation == 'SEvsLFC':
          gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation )
          storageElementName = subRequestAttributes['StorageElement']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile['LFN']
              storageElement = StorageElement( storageElementName )
              res = storageElement.isValid()
              if not res['OK']:
                errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
                gLogger.error( errStr, storageElement )
              else:
                res = storageElement.getPfnForLfn( lfn )
                if not res['OK']:
                  gLogger.info( 'shit bugger do something.' )
                else:
                  oNamespaceBrowser = NamespaceBrowser( res['Value'] )
                  # Loop over all the directories and sub-directories
                  while ( oNamespaceBrowser.isActive() ):
                    currentDir = oNamespaceBrowser.getActiveDir()

                    gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir )
                    res = storageElement.listDirectory( currentDir )
                    if not res['Value']['Successful'].has_key( currentDir ):
                      gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) )
                      subDirs = [currentDir]
                    else:
                      subDirs = []
                      files = {}
                      for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items():
                        pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
                        surl = pfnRes['Value']
                        files[surl] = surlDict
                      for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items():
                        pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
                        surl = pfnRes['Value']
                        subDirs.append( surl )

                      #subDirs = res['Value']['Successful'][currentDir]['SubDirs']
                      gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) )
                      #files = res['Value']['Successful'][currentDir]['Files']
                      gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) )

                      selectedLfns = []
                      lfnPfnDict = {}
                      pfnSize = {}

                      for pfn, pfnDict in files.items():
                        res = storageElement.getPfnPath( pfn )
                        if not res['OK']:
                          gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) )
                          fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']}
                          res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
                          if res['OK']:
                            gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
                            gLogger.error( "Change the status in the LFC,ProcDB...." )
                          else:
                            gLogger.error( "Shit, fuck, bugger. Add the failover." )
                        else:
                          lfn = res['Value']
                          selectedLfns.append( lfn )
                          lfnPfnDict[lfn] = pfn
                          pfnSize[pfn] = pfnDict['Size']
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]
  def executeRequest( self ):
    ################################################
    # Get a request from request DB
    res = self.RequestDBClient.getRequest( 'register' )
    if not res['OK']:
      gLogger.info( "RegistrationAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "RegistrationAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    try:
      jobID = int( res['Value']['JobID'] )
    except:
      jobID = 0
    gLogger.info( "RegistrationAgent.execute: Obtained request %s" % requestName )

    result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
    if result['OK']:
      currentOrder = result['Value']
    else:
      return S_OK( 'Can not get the request execution order' )

    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'register' )
    if not res['OK']:
      errStr = "RegistrationAgent.execute: Failed to obtain number of transfer subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "RegistrationAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    modified = False
    for ind in range( res['Value'] ):
      gLogger.info( "RegistrationAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'register' )['Value']
      subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
      subStatus = subRequestAttributes['Status']
      if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'register' )['Value']
        operation = subRequestAttributes['Operation']

        ################################################
        #  If the sub-request is a register file operation
        if operation == 'registerFile':
          gLogger.info( "RegistrationAgent.execute: Attempting to execute %s sub-request." % operation )
          diracSE = str( subRequestAttributes['TargetSE'] )
          if diracSE == 'SE':
            # We do not care about SE, put any there
            diracSE = "CERN-FAILOVER"
          catalog = subRequestAttributes['Catalogue']
          if catalog == "None":
            catalog = ''
          subrequest_done = True
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              lfn = subRequestFile.get( 'LFN', '' )
              if lfn: lfn = str( lfn )
              physicalFile = subRequestFile.get( 'PFN', '' )
              if physicalFile: physicalFile = str( physicalFile )
              fileSize = subRequestFile.get( 'Size', 0 )
              if fileSize: fileSize = int( fileSize )
              fileGuid = subRequestFile.get( 'GUID', '' )
              if fileGuid: fileGuid = str( fileGuid )
              checksum = subRequestFile.get( 'Addler', '' )
              if checksum: checksum = str( checksum )
              if catalog == 'BookkeepingDB':
                diracSE = 'CERN-HIST'
              fileTuple = ( lfn, physicalFile, fileSize, diracSE, fileGuid, checksum )
              res = self.ReplicaManager.registerFile( fileTuple, catalog )
              print res
              if not res['OK']:
                self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
                errStr = "RegistrationAgent.execute: Completely failed to register file."
                gLogger.error( errStr, res['Message'] )
                subrequest_done = False
              elif lfn in res['Value']['Failed'].keys():
                self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
                errStr = "RegistrationAgent.execute: Completely failed to register file."
                gLogger.error( errStr, res['Value']['Failed'][lfn] )
                subrequest_done = False
              else:
                self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
                oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                modified = True
            else:
              gLogger.info( "RegistrationAgent.execute: File already completed." )
          if subrequest_done:
            oRequest.setSubRequestStatus( ind, 'register', 'Done' )

        ################################################
        #  If the sub-request is none of the above types
        else:
          gLogger.error( "RegistrationAgent.execute: Operation not supported.", operation )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]

#.........这里部分代码省略.........

      ######################################################################################
      #
      #  The important request attributes are the source and target SEs.
      #

      for ind in range( numberRequests ):
        gLogger.info( "ReplicationScheduler._execute: Treating sub-request %s from '%s'." % ( ind, requestName ) )
        attributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
        if attributes['Status'] != 'Waiting':
          #  If the sub-request is already in terminal state
          gLogger.info( "ReplicationScheduler._execute: Sub-request %s is status '%s' and  not to be executed." % ( ind, attributes['Status'] ) )
          continue

        sourceSE = attributes['SourceSE']
        targetSE = attributes['TargetSE']
        """ This section should go in the transfer request class """
        if type( targetSE ) in types.StringTypes:
          if re.search( ',', targetSE ):
            targetSEs = targetSE.split( ',' )
          else:
            targetSEs = [targetSE]
        """----------------------------------------------------- """
        operation = attributes['Operation']
        reqRepStrategy = None
        if operation in self.strategyHandler.getSupportedStrategies():
          reqRepStrategy = operation

        ######################################################################################
        #
        # Then obtain the file attribute of interest are the  LFN and FileID
        #

        res = oRequest.getSubRequestFiles( ind, 'transfer' )
        if not res['OK']:
          gLogger.error( 'ReplicationScheduler._execute: Failed to obtain sub-request files.' , res['Message'] )
          continue
        files = res['Value']
        gLogger.info( "ReplicationScheduler._execute: Sub-request %s found with %s files." % ( ind, len( files ) ) )
        filesDict = {}
        for file in files:
          lfn = file['LFN']
          if file['Status'] != 'Waiting':
            gLogger.debug( "ReplicationScheduler._execute: %s will not be scheduled because it is %s." % ( lfn, file['Status'] ) )
          else:
            fileID = file['FileID']
            filesDict[lfn] = fileID
        if not filesDict:
          gLogger.info( "ReplicationScheduler._execute: No Waiting files found for request" )
          continue
        notSched = len( files ) - len( filesDict )
        if notSched:
          gLogger.info( "ReplicationScheduler._execute: %d files found not Waiting" % notSched )

        ######################################################################################
        #
        #  Now obtain replica information for the files associated to the sub-request.
        #

        lfns = filesDict.keys()
        gLogger.info( "ReplicationScheduler._execute: Obtaining replica information for %d sub-request files." % len( lfns ) )
        res = self.rm.getCatalogReplicas( lfns )
        if not res['OK']:
          gLogger.error( "ReplicationScheduler._execute: Failed to get replica information.", res['Message'] )
          continue
        for lfn, failure in res['Value']['Failed'].items():

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]

#.........这里部分代码省略.........
          self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
          failover[fileName] = metadata
        else:
          lfn = metadata['lfn']
    else:
      failover = final

    self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)  

    cleanUp = False
    for fileName, metadata in failover.items():
      self.log.info('Setting default catalog for failover transfer to FileCatalog')
      random.shuffle(self.failoverSEs)
      targetSE = metadata['resolvedSE'][0]
      metadata['resolvedSE'] = self.failoverSEs
      result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'],
                                                                metadata['lfn'], targetSE, metadata['resolvedSE'],
                                                                fileGUID = metadata['guid'], fileCatalog = catalogs)
      if not result['OK']:
        self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
        cleanUp = True
        break #no point continuing if one completely fails

    os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess

    #Now after all operations, retrieve potentially modified request object
    result = failoverTransfer.getRequestObject()
    if not result['OK']:
      self.log.error(result)
      return S_ERROR('Could not retrieve modified request')

    self.request = result['Value']

    #If some or all of the files failed to be saved to failover
    if cleanUp:
      lfns = []
      for fileName, metadata in final.items():
        lfns.append(metadata['lfn'])

      result = self.__cleanUp(lfns)
      self.workflow_commons['Request'] = self.request
      return S_ERROR('Failed to upload output data')

#    #Can now register the successfully uploaded files in the BK
#    if not performBKRegistration:
#      self.log.info('There are no files to perform the BK registration for, all could be saved to failover')
#    else:
#      rm = ReplicaManager()
#      result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB'])
#      self.log.verbose(result)
#      if not result['OK']:
#        self.log.error(result)
#        return S_ERROR('Could Not Perform BK Registration')
#      if result['Value']['Failed']:
#        for lfn,error in result['Value']['Failed'].items():
#          self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error))
#          result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register')
#          if not result['OK']:
#            self.log.error('Could not set registerFile request:\n%s' %result)
#            return S_ERROR('Could Not Set BK Registration Request')
#          fileDict = {'LFN':lfn,'Status':'Waiting'}
#          index = result['Value']
#          self.request.setSubRequestFiles(index,'register',[fileDict])

    self.workflow_commons['Request'] = self.request
    return S_OK('Output data uploaded')

  #############################################################################
  def __cleanUp(self, lfnList):
    """ Clean up uploaded data for the LFNs in the list
    """
    # Clean up the current request
    for req_type in ['transfer', 'register']:
      for lfn in lfnList:
        result = self.request.getNumSubRequests(req_type)
        if result['OK']:
          nreq = result['Value']
          if nreq:
            # Go through subrequests in reverse order in order not to spoil the numbering
            ind_range = [0]
            if nreq > 1:
              ind_range = range(nreq-1, -1, -1)
            for i in ind_range:
              result = self.request.getSubRequestFiles(i, req_type)
              if result['OK']:
                fileList = result['Value']
                if fileList[0]['LFN'] == lfn:
                  result = self.request.removeSubRequest(i, req_type)

    # Set removal requests just in case
    for lfn in lfnList:
      result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '',
                                                          'ExecutionOrder' : 1}}, 'removal')
      index = result['Value']
      fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'}
      self.request.setSubRequestFiles(index, 'removal', [fileDict])

    return S_OK()

#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]
  def executeRequest( self ):
    """
    Do the actual work in the Thread
    """
    ################################################
    # Get a request from request DB
    gMonitor.addMark( "Iteration", 1 )
    res = self.requestDBClient.getRequest( 'removal' )
    if not res['OK']:
      gLogger.info( "RemovalAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "RemovalAgent.execute: No requests to be executed found." )
      self.pendingRequests = False
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    try:
      jobID = int( res['Value']['JobID'] )
    except ValueError:
      jobID = 0
    gLogger.info( "RemovalAgent.execute: Obtained request %s" % requestName )

    try:

      result = self.requestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
      if result['OK']:
        currentOrder = result['Value']
      else:
        gLogger.error( 'Can not get the request execution order' )
        self.requestDBClient.updateRequest( requestName, requestString, sourceServer )
        return S_OK( 'Can not get the request execution order' )

      oRequest = RequestContainer( request = requestString )

      ################################################
      # Find the number of sub-requests from the request
      res = oRequest.getNumSubRequests( 'removal' )
      if not res['OK']:
        errStr = "RemovalAgent.execute: Failed to obtain number of removal subrequests."
        gLogger.error( errStr, res['Message'] )
        return S_OK()
      gLogger.info( "RemovalAgent.execute: Found %s sub requests." % res['Value'] )

      ################################################
      # For all the sub-requests in the request
      modified = False
      for ind in range( res['Value'] ):
        gMonitor.addMark( "Execute", 1 )
        gLogger.info( "RemovalAgent.execute: Processing sub-request %s." % ind )
        subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'removal' )['Value']
        subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
        subStatus = subRequestAttributes['Status']
        if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
          subRequestFiles = oRequest.getSubRequestFiles( ind, 'removal' )['Value']
          operation = subRequestAttributes['Operation']

          ################################################
          #  If the sub-request is a physical removal operation
          if operation == 'physicalRemoval':
            gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
            diracSEs = subRequestAttributes['TargetSE'].split( ',' )
            physicalFiles = []
            pfnToLfn = {}
            for subRequestFile in subRequestFiles:
              if subRequestFile['Status'] == 'Waiting':
                pfn = str( subRequestFile['PFN'] )
                lfn = str( subRequestFile['LFN'] )
                pfnToLfn[pfn] = lfn
                physicalFiles.append( pfn )
            gMonitor.addMark( 'PhysicalRemovalAtt', len( physicalFiles ) )
            failed = {}
            errMsg = {}
            for diracSE in diracSEs:
              res = self.replicaManager.removeStorageFile( physicalFiles, diracSE )
              if res['OK']:
                for pfn in res['Value']['Failed'].keys():
                  if not failed.has_key( pfn ):
                    failed[pfn] = {}
                  failed[pfn][diracSE] = res['Value']['Failed'][pfn]
              else:
                errMsg[diracSE] = res['Message']
                for pfn in physicalFiles:
                  if not failed.has_key( pfn ):
                    failed[pfn] = {}
                  failed[pfn][diracSE] = 'Completely'
            # Now analyse the results
            failedPFNs = failed.keys()
            pfnsOK = [pfn for pfn in physicalFiles if not pfn in failedPFNs]
            gMonitor.addMark( 'PhysicalRemovalDone', len( pfnsOK ) )
            for pfn in pfnsOK:
              gLogger.info( "RemovalAgent.execute: Successfully removed %s at %s" % ( pfn, str( diracSEs ) ) )
              res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', pfnToLfn[pfn], 'Status', 'Done' )
              if not res['OK']:
                gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', pfnToLfn[pfn] ) )
              modified = True
            if failed:
              gMonitor.addMark( 'PhysicalRemovalFail', len( failedPFNs ) )
              for pfn in failedPFNs:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import getSubRequestFiles [as 别名]
  def executeRequest( self ):
    ################################################
    # Get a request from request DB
    gMonitor.addMark( "Iteration", 1 )
    res = self.RequestDBClient.getRequest( 'transfer' )
    if not res['OK']:
      gLogger.info( "TransferAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "TransferAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    try:
      jobID = int( res['Value']['JobID'] )
    except:
      jobID = 0
    gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName )

    result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
    if result['OK']:
      currentOrder = result['Value']
    else:
      return S_OK( 'Can not get the request execution order' )

    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'transfer' )
    if not res['OK']:
      errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    modified = False
    for ind in range( res['Value'] ):
      gMonitor.addMark( "Execute", 1 )
      gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
      if subRequestAttributes['ExecutionOrder']:
        subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
      else:
        subExecutionOrder = 0
      subStatus = subRequestAttributes['Status']
      if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value']
        operation = subRequestAttributes['Operation']

        subRequestError = ''       
        ################################################
        #  If the sub-request is a put and register operation
        if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
          gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
          diracSE = str( subRequestAttributes['TargetSE'] )
          catalog = ''
          if  subRequestAttributes.has_key( 'Catalogue' ):
            catalog = subRequestAttributes['Catalogue']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              gMonitor.addMark( "Put and register", 1 )
              lfn = str( subRequestFile['LFN'] )
              file = subRequestFile['PFN']
              guid = subRequestFile['GUID']
              addler = subRequestFile['Addler']
              res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog )
              if res['OK']:
                if res['Value']['Successful'].has_key( lfn ):
                  if not res['Value']['Successful'][lfn].has_key( 'put' ):
                    gMonitor.addMark( "Put failed", 1 )
                    self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
                    gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) )
                    subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
                  elif not res['Value']['Successful'][lfn].has_key( 'register' ):
                    gMonitor.addMark( "Put successful", 1 )
                    gMonitor.addMark( "File registration failed", 1 )
                    self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
                    self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' )
                    gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
                    gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' )
                    subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE )
                    fileDict = res['Value']['Failed'][lfn]['register']
                    registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'}, 
                                           'Files':[{'LFN': fileDict['LFN'], 
                                                     'PFN':fileDict['PFN'], 
                                                     'Size':fileDict['Size'], 
                                                     'Addler':fileDict['Addler'], 
                                                     'GUID':fileDict['GUID']}]}
                    gLogger.info( "TransferAgent.execute: Setting registration request for failed file." )
                    oRequest.addSubRequest( registerRequestDict, 'register' )
                    modified = True
                  else:
                    gMonitor.addMark( "Put successful", 1 )
#.........这里部分代码省略.........