Python RequestContainer.addSubRequest方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  def generateRequest(self):
    """ Generate failover requests for the operations in the internal cache
    """

    request = RequestContainer()
    result = self.sendStoredStatusInfo()

    if not result['OK']:
      if result.has_key('rpcStub'):
        request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset')
      else:
        return S_ERROR('Could not create job state sub-request')

    result = self.sendStoredJobParameters()

    if not result['OK']:
      if result.has_key('rpcStub'):
        request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset')
      else:
        return S_ERROR('Could not create job parameters sub-request')

    if request.isEmpty()['Value']:
      request = None

    return S_OK(request)

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
    """ Set removal request with the given credentials
    """
    request = RequestContainer()
    request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
    requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
    request.setRequestName( requestName )
    request.setSourceComponent( 'JobCleaningAgent' )

    removalDict = {'Attributes':{ 'Operation':'removeFile',
                                  'TargetSE':'',
                                  'ExecutionOrder':0
                                }
                   }
    result = request.addSubRequest( removalDict, 'removal' )
    if not result['OK']:
      return result

    index = result['Value']
    fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
    request.setSubRequestFiles( index, 'removal', [fileDict] )

    client = RequestClient()
    result = request.toXML()
    if not result['OK']:
      return result
    xmlRequest = result['Value']
    result = client.setRequest( requestName, xmlRequest )
    return result

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  def test__getLastOrder(self):
    # no files
    req = RequestContainer()
    res = req._getLastOrder()
    self.assertEqual( res, 0 )

    self.assertEqual( req.subRequests, {} )

    req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
                                      'TargetSE':'SE', 'ExecutionOrder': 0}},
                      'transfer' )
    res = req._getLastOrder()
    self.assertEqual( res, 0 )

    req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
                                      'TargetSE':'SE', 'ExecutionOrder': 1}},
                      'transfer' )
    res = req._getLastOrder()
    self.assertEqual( res, 1 )

    del( req )

    # with files
    req = RequestContainer()
    res = req._getLastOrder( 'foo' )
    self.assertEqual( res, 0 )

    req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
                                      'TargetSE':'SE', 'ExecutionOrder': 1}},
                      'transfer' )
    res = req._getLastOrder( 'foo' )
    self.assertEqual( res, 0 )

    req.setSubRequestFiles( 0, 'transfer', [{'LFN':'foo', 'Status':'Waiting'}] )
    res = req._getLastOrder( 'foo' )
    self.assertEqual( res, 1 )


    req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
                                      'TargetSE':'SE', 'ExecutionOrder': 2}},
                      'removal' )
    res = req._getLastOrder( 'foo' )
    self.assertEqual( res, 1 )

    req.setSubRequestFiles( 0, 'removal', [{'LFN':'foo', 'Status':'Waiting'}] )
    res = req._getLastOrder( 'foo' )
    self.assertEqual( res, 2 )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
    def test__getLastOrder(self):
        # no files
        req = RequestContainer()
        res = req._getLastOrder()
        self.assertEqual(res, 0)

        self.assertEqual(req.subRequests, {})

        req.addSubRequest(
            {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 0}}, "transfer"
        )
        res = req._getLastOrder()
        self.assertEqual(res, 0)

        req.addSubRequest(
            {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer"
        )
        res = req._getLastOrder()
        self.assertEqual(res, 1)

        del (req)

        # with files
        req = RequestContainer()
        res = req._getLastOrder("foo")
        self.assertEqual(res, 0)

        req.addSubRequest(
            {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer"
        )
        res = req._getLastOrder("foo")
        self.assertEqual(res, 0)

        req.setSubRequestFiles(0, "transfer", [{"LFN": "foo", "Status": "Waiting"}])
        res = req._getLastOrder("foo")
        self.assertEqual(res, 1)

        req.addSubRequest(
            {"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 2}}, "removal"
        )
        res = req._getLastOrder("foo")
        self.assertEqual(res, 1)

        req.setSubRequestFiles(0, "removal", [{"LFN": "foo", "Status": "Waiting"}])
        res = req._getLastOrder("foo")
        self.assertEqual(res, 2)

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
 def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
   if self.getCSOption( "DelayedExternalDeletion", True ):
     gLogger.info( "Setting deletion request" )
     try:
       request = RequestContainer()
       result = request.addSubRequest( { 'Attributes' : { 'Operation' : 'removePhysicalFile',
                                                          'TargetSE' : SEName,
                                                          'ExecutionOrder' : 1
                                                         } },
                                        'removal' )
       index = result['Value']
       fileDict = { 'PFN' : SEPFN, 'Status' : 'Waiting' }
       request.setSubRequestFiles( index, 'removal', [ fileDict ] )
       return RequestClient().setRequest( "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ),
                                          request.toXML()[ 'Value' ] )
     except Exception, e:
       gLogger.exception( "Exception while setting deletion request" )
       return S_ERROR( "Cannot set deletion request: %s" % str( e ) )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  def test_addSubRequest( self ):
    rc_o = RequestContainer()
    op1_Index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op1'}},
                                    'someType' )
    op1_Index = op1_Index['Value']
    subRequestExpected = {'someType': [{'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                                    'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                                    'Catalogue': '', 'Error': '', 'Operation': 'op1'},
                                        'Datasets': []}]}
    self.assertEqual( rc_o.subRequests, subRequestExpected )

    op2_index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op2'}},
                                    'someType' )
    op2_index = op2_index['Value']
    subRequestExpected = {
                          'someType':
                                      [
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op1'},
                                        'Datasets': []
                                        },
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op2'},
                                        'Datasets': []
                                        }
                                      ]
                          }
    self.assertEqual( rc_o.subRequests, subRequestExpected )

    rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
                        'someType' )
    subRequestExpected = {
                          'someType':
                                      [
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op1'},
                                        'Datasets': []
                                        },
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op2'},
                                        'Datasets': []
                                        },
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': ''},
                                        'Datasets': []
                                        }
                                      ]
                          }
    self.assertEqual( rc_o.subRequests, subRequestExpected )


    rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
                        'someOtherType' )
    subRequestExpected = {
                          'someType':
                                      [
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op1'},
                                        'Datasets': []
                                        },
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': 'op2'},
                                        'Datasets': []
                                        },
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': ''},
                                        'Datasets': []
                                        }
                                      ],
                          'someOtherType':
                                      [
                                       {
                                        'Files': [],
                                        'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
                                                       'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
                                                       'Catalogue': '', 'Error': '', 'Operation': ''},
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  else:
    lfnpath = "%s%s" % (path, os.path.basename(appTar))
    res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM"))
    if not res['OK']:
      return res
    request = RequestContainer()
    request.setCreationTime()
    requestClient = RequestClient()
    request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz",""))
    request.setSourceComponent('ReplicateILCSoft')
    copies_at = ops.getValue('Software/CopiesAt',[])
    index_copy = 0
    for copies in copies_at:
      res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister',
                                                 'TargetSE' : copies,
                                                 'ExecutionOrder' : index_copy},
                                   'Files':[{'LFN':lfnpath}]},
                                   'transfer')
      #res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM")
      if not res['OK']:
        return res
      index_copy += 1
    requestxml = request.toXML()['Value']
    if copies_at:
      res = requestClient.setRequest(request.getRequestName()['Value'], requestxml)
      if not res['OK']:
        gLogger.error('Could not set replication request %s' % res['Message'])
      return S_OK('Application uploaded')
  return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]

#.........这里部分代码省略.........
        metadata['filedict']=fileDict
        metadata['uploadedSE']=se
        metadata['lfn']=lfn
        metadata['registration']='request'
        return S_OK(metadata)

    self.log.error('Encountered %s errors during attempts to upload output data' %len(errorList))
    return S_ERROR('Failed to upload output data file')

  #############################################################################
  def transferAndRegisterFileFailover(self,fileName,localPath,lfn,targetSE,failoverSEList,fileGUID=None,fileCatalog=None):
    """Performs the transfer and register operation to failover storage and sets the
       necessary replication and removal requests to recover.
    """
    failover = self.transferAndRegisterFile(fileName,localPath,lfn,failoverSEList,fileGUID,fileCatalog)
    if not failover['OK']:
      self.log.error('Could not upload file to failover SEs',failover['Message'])
      return failover

    #set removal requests and replication requests
    result = self.__setFileReplicationRequest(lfn,targetSE)
    if not result['OK']:
      self.log.error('Could not set file replication request',result['Message'])
      return result

    lfn = failover['Value']['lfn']
    failoverSE = failover['Value']['uploadedSE']
    self.log.info('Attempting to set replica removal request for LFN %s at failover SE %s' %(lfn,failoverSE))
    result = self.__setReplicaRemovalRequest(lfn,failoverSE)
    if not result['OK']:
      self.log.error('Could not set removal request',result['Message'])
      return result

    return S_OK('%s uploaded to a failover SE' %fileName)

  #############################################################################
  def getRequestObject(self):
    """Returns the potentially modified request object in order to propagate changes.
    """
    return S_OK(self.request)
  
  #############################################################################
  def __setFileReplicationRequest(self,lfn,se):
    """ Sets a registration request.
    """
    self.log.info('Setting replication request for %s to %s' % (lfn,se))
    result = self.request.addSubRequest({'Attributes':{'Operation':'replicateAndRegister',
                                                       'TargetSE':se,'ExecutionOrder':0}},
                                         'transfer')
    if not result['OK']:
      return result

    index = result['Value']
    fileDict = {'LFN':lfn,'Status':'Waiting'}
    self.request.setSubRequestFiles(index,'transfer',[fileDict])

    return S_OK()

  #############################################################################
  def __setRegistrationRequest(self,lfn,se,catalog,fileDict):
    """ Sets a registration request.
    """
    self.log.info('Setting registration request for %s at %s.' % (lfn,se))
    result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0,
                                                       'TargetSE':se,'Catalogue':catalog}},'register')
    if not result['OK']:
      return result

    index = result['Value']
    if not fileDict.has_key('Status'):
      fileDict['Status']='Waiting'
    self.request.setSubRequestFiles(index,'register',[fileDict])

    return S_OK()

  #############################################################################
  def __setReplicaRemovalRequest(self,lfn,se):
    """ Sets a removal request for a replica.
    """
    result = self.request.addSubRequest({'Attributes':{'Operation':'replicaRemoval',
                                                       'TargetSE':se,'ExecutionOrder':1}},
                                         'removal')
    index = result['Value']
    fileDict = {'LFN':lfn,'Status':'Waiting'}
    self.request.setSubRequestFiles(index,'removal',[fileDict])

    return S_OK()

  #############################################################################
  def __setFileRemovalRequest(self,lfn,se='',pfn=''):
    """ Sets a removal request for a file including all replicas.
    """
    result = self.request.addSubRequest({'Attributes':{'Operation':'removeFile',
                                                       'TargetSE':se,'ExecutionOrder':1}},
                                         'removal')
    index = result['Value']
    fileDict = {'LFN':lfn,'PFN':pfn,'Status':'Waiting'}
    self.request.setSubRequestFiles(index,'removal',[fileDict])

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]

#.........这里部分代码省略.........
          self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
          failover[fileName] = metadata
        else:
          lfn = metadata['lfn']
    else:
      failover = final

    self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)  

    cleanUp = False
    for fileName, metadata in failover.items():
      self.log.info('Setting default catalog for failover transfer to FileCatalog')
      random.shuffle(self.failoverSEs)
      targetSE = metadata['resolvedSE'][0]
      metadata['resolvedSE'] = self.failoverSEs
      result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'],
                                                                metadata['lfn'], targetSE, metadata['resolvedSE'],
                                                                fileGUID = metadata['guid'], fileCatalog = catalogs)
      if not result['OK']:
        self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
        cleanUp = True
        break #no point continuing if one completely fails

    os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess

    #Now after all operations, retrieve potentially modified request object
    result = failoverTransfer.getRequestObject()
    if not result['OK']:
      self.log.error(result)
      return S_ERROR('Could not retrieve modified request')

    self.request = result['Value']

    #If some or all of the files failed to be saved to failover
    if cleanUp:
      lfns = []
      for fileName, metadata in final.items():
        lfns.append(metadata['lfn'])

      result = self.__cleanUp(lfns)
      self.workflow_commons['Request'] = self.request
      return S_ERROR('Failed to upload output data')

#    #Can now register the successfully uploaded files in the BK
#    if not performBKRegistration:
#      self.log.info('There are no files to perform the BK registration for, all could be saved to failover')
#    else:
#      rm = ReplicaManager()
#      result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB'])
#      self.log.verbose(result)
#      if not result['OK']:
#        self.log.error(result)
#        return S_ERROR('Could Not Perform BK Registration')
#      if result['Value']['Failed']:
#        for lfn,error in result['Value']['Failed'].items():
#          self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error))
#          result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register')
#          if not result['OK']:
#            self.log.error('Could not set registerFile request:\n%s' %result)
#            return S_ERROR('Could Not Set BK Registration Request')
#          fileDict = {'LFN':lfn,'Status':'Waiting'}
#          index = result['Value']
#          self.request.setSubRequestFiles(index,'register',[fileDict])

    self.workflow_commons['Request'] = self.request
    return S_OK('Output data uploaded')

  #############################################################################
  def __cleanUp(self, lfnList):
    """ Clean up uploaded data for the LFNs in the list
    """
    # Clean up the current request
    for req_type in ['transfer', 'register']:
      for lfn in lfnList:
        result = self.request.getNumSubRequests(req_type)
        if result['OK']:
          nreq = result['Value']
          if nreq:
            # Go through subrequests in reverse order in order not to spoil the numbering
            ind_range = [0]
            if nreq > 1:
              ind_range = range(nreq-1, -1, -1)
            for i in ind_range:
              result = self.request.getSubRequestFiles(i, req_type)
              if result['OK']:
                fileList = result['Value']
                if fileList[0]['LFN'] == lfn:
                  result = self.request.removeSubRequest(i, req_type)

    # Set removal requests just in case
    for lfn in lfnList:
      result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '',
                                                          'ExecutionOrder' : 1}}, 'removal')
      index = result['Value']
      fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'}
      self.request.setSubRequestFiles(index, 'removal', [fileDict])

    return S_OK()

#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
    def test_addSubRequest(self):
        rc_o = RequestContainer()
        op1_Index = rc_o.addSubRequest(
            {"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op1"}},
            "someType",
        )
        op1_Index = op1_Index["Value"]
        subRequestExpected = {
            "someType": [
                {
                    "Files": [],
                    "Attributes": {
                        "Status": "Waiting",
                        "LastUpdate": "",
                        "TargetSE": "",
                        "ExecutionOrder": 0,
                        "SubRequestID": "x",
                        "CreationTime": "2012-06-06 14:53:43.763743",
                        "Catalogue": "",
                        "Error": "",
                        "Operation": "op1",
                    },
                    "Datasets": [],
                }
            ]
        }
        self.assertEqual(rc_o.subRequests, subRequestExpected)

        op2_index = rc_o.addSubRequest(
            {"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op2"}},
            "someType",
        )
        op2_index = op2_index["Value"]
        subRequestExpected = {
            "someType": [
                {
                    "Files": [],
                    "Attributes": {
                        "Status": "Waiting",
                        "LastUpdate": "",
                        "TargetSE": "",
                        "ExecutionOrder": 0,
                        "SubRequestID": "x",
                        "CreationTime": "2012-06-06 14:53:43.763743",
                        "Catalogue": "",
                        "Error": "",
                        "Operation": "op1",
                    },
                    "Datasets": [],
                },
                {
                    "Files": [],
                    "Attributes": {
                        "Status": "Waiting",
                        "LastUpdate": "",
                        "TargetSE": "",
                        "ExecutionOrder": 0,
                        "SubRequestID": "x",
                        "CreationTime": "2012-06-06 14:53:43.763743",
                        "Catalogue": "",
                        "Error": "",
                        "Operation": "op2",
                    },
                    "Datasets": [],
                },
            ]
        }
        self.assertEqual(rc_o.subRequests, subRequestExpected)

        rc_o.addSubRequest(
            {
                "Attributes": {
                    "SubRequestID": "x",
                    "CreationTime": "2012-06-06 14:53:43.763743",
                    "ExecutionOrder": "last",
                }
            },
            "someType",
        )
        subRequestExpected = {
            "someType": [
                {
                    "Files": [],
                    "Attributes": {
                        "Status": "Waiting",
                        "LastUpdate": "",
                        "TargetSE": "",
                        "ExecutionOrder": 0,
                        "SubRequestID": "x",
                        "CreationTime": "2012-06-06 14:53:43.763743",
                        "Catalogue": "",
                        "Error": "",
                        "Operation": "op1",
                    },
                    "Datasets": [],
                },
                {
                    "Files": [],
                    "Attributes": {
                        "Status": "Waiting",
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.RequestContainer.RequestContainer import addSubRequest [as 别名]
  def executeRequest( self ):
    ################################################
    # Get a request from request DB
    gMonitor.addMark( "Iteration", 1 )
    res = self.RequestDBClient.getRequest( 'transfer' )
    if not res['OK']:
      gLogger.info( "TransferAgent.execute: Failed to get request from database." )
      return S_OK()
    elif not res['Value']:
      gLogger.info( "TransferAgent.execute: No requests to be executed found." )
      return S_OK()
    requestString = res['Value']['RequestString']
    requestName = res['Value']['RequestName']
    sourceServer = res['Value']['Server']
    try:
      jobID = int( res['Value']['JobID'] )
    except:
      jobID = 0
    gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName )

    result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
    if result['OK']:
      currentOrder = result['Value']
    else:
      return S_OK( 'Can not get the request execution order' )

    oRequest = RequestContainer( request = requestString )

    ################################################
    # Find the number of sub-requests from the request
    res = oRequest.getNumSubRequests( 'transfer' )
    if not res['OK']:
      errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
      gLogger.error( errStr, res['Message'] )
      return S_OK()
    gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] )

    ################################################
    # For all the sub-requests in the request
    modified = False
    for ind in range( res['Value'] ):
      gMonitor.addMark( "Execute", 1 )
      gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind )
      subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
      if subRequestAttributes['ExecutionOrder']:
        subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
      else:
        subExecutionOrder = 0
      subStatus = subRequestAttributes['Status']
      if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
        subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value']
        operation = subRequestAttributes['Operation']

        subRequestError = ''       
        ################################################
        #  If the sub-request is a put and register operation
        if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
          gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
          diracSE = str( subRequestAttributes['TargetSE'] )
          catalog = ''
          if  subRequestAttributes.has_key( 'Catalogue' ):
            catalog = subRequestAttributes['Catalogue']
          for subRequestFile in subRequestFiles:
            if subRequestFile['Status'] == 'Waiting':
              gMonitor.addMark( "Put and register", 1 )
              lfn = str( subRequestFile['LFN'] )
              file = subRequestFile['PFN']
              guid = subRequestFile['GUID']
              addler = subRequestFile['Addler']
              res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog )
              if res['OK']:
                if res['Value']['Successful'].has_key( lfn ):
                  if not res['Value']['Successful'][lfn].has_key( 'put' ):
                    gMonitor.addMark( "Put failed", 1 )
                    self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
                    gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) )
                    subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
                  elif not res['Value']['Successful'][lfn].has_key( 'register' ):
                    gMonitor.addMark( "Put successful", 1 )
                    gMonitor.addMark( "File registration failed", 1 )
                    self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
                    self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' )
                    gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
                    gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
                    oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' )
                    subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE )
                    fileDict = res['Value']['Failed'][lfn]['register']
                    registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'}, 
                                           'Files':[{'LFN': fileDict['LFN'], 
                                                     'PFN':fileDict['PFN'], 
                                                     'Size':fileDict['Size'], 
                                                     'Addler':fileDict['Addler'], 
                                                     'GUID':fileDict['GUID']}]}
                    gLogger.info( "TransferAgent.execute: Setting registration request for failed file." )
                    oRequest.addSubRequest( registerRequestDict, 'register' )
                    modified = True
                  else:
                    gMonitor.addMark( "Put successful", 1 )
#.........这里部分代码省略.........