Python File.File类代码示例

  def __insertRegisterOperation( self, request, operation, toRegister ):
    """ add RegisterReplica operation

    :param Request request: request instance
    :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
    :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
    """
    log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )

    byTarget = {}
    for ftsFile in toRegister:
      if ftsFile.TargetSE not in byTarget:
        byTarget.setdefault( ftsFile.TargetSE, [] )
      byTarget[ftsFile.TargetSE].append( ftsFile )
    log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )

    for target, ftsFileList in byTarget.iteritems():
      log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
                                                                                            len( ftsFileList ) ) )
      registerOperation = Operation()
      registerOperation.Type = "RegisterReplica"
      registerOperation.Status = "Waiting"
      registerOperation.TargetSE = target
      targetSE = StorageElement( target )
      for ftsFile in ftsFileList:
        opFile = File()
        opFile.LFN = ftsFile.LFN
        pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
        if not pfn["OK"]:
          continue
        opFile.PFN = pfn["Value"]
        registerOperation.addFile( opFile )
      request.insertBefore( registerOperation, operation )

    return S_OK()

  def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
    """ Set a request to upload job log files from the output sandbox
        Changed to be similar to LHCb createLogUploadRequest
        using LHCb LogUpload Request and Removal Request
    """
    self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
    request = self._getRequestContainer()

    logUpload = Operation()
    logUpload.Type = "LogUpload"
    logUpload.TargetSE = targetSE

    upFile = File()
    upFile.LFN = logFileLFN
    logUpload.addFile( upFile )

    logRemoval = Operation()
    logRemoval.Type = 'RemoveFile'
    logRemoval.TargetSE = uploadedSE
    logRemoval.addFile( upFile )

    request.addOperation ( logUpload )
    request.addOperation ( logRemoval )

    self.workflow_commons['Request'] = request

    return S_OK()

  def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
    if self.getCSOption( "DelayedExternalDeletion", True ):
      gLogger.info( "Setting deletion request" )
      try:

        request = Request()
        request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
        physicalRemoval = Operation()
        physicalRemoval.Type = "PhysicalRemoval"
        physicalRemoval.TargetSE = SEName
        fileToRemove = File()
        fileToRemove.PFN = SEPFN
        physicalRemoval.addFile( fileToRemove )
        request.addOperation( physicalRemoval )
        return ReqClient().putRequest( request )
      except Exception as e:
        gLogger.exception( "Exception while setting deletion request" )
        return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
    else:
      gLogger.info( "Deleting external Sandbox" )
      try:
        return StorageElement( SEName ).removeFile( SEPFN )
      except Exception as e:
        gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
        return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )

def myRequest():
  """Create a request and put it to the db"""

  request = Request()
  request.RequestName = 'myAwesomeRemovalRequest.xml'
  request.JobID = 0
  request.SourceComponent = "myScript"

  remove = Operation()
  remove.Type = "RemoveFile"

  lfn = "/ilc/user/s/sailer/test.txt"
  rmFile = File()
  rmFile.LFN = lfn
  remove.addFile( rmFile )

  request.addOperation( remove )
  isValid = RequestValidator().validate( request )
  if not isValid['OK']:
    raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
  else:
    print("It is a GOGOGO")
    requestClient = ReqClient()
    result = requestClient.putRequest( request )
    print(result)

  def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
    """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
    """
    if ( not owner ) or ( not ownerGroup ):
      res = getProxyInfo( False, False )
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    if not ownerDN:
      res = getDNForUsername( owner )
      if not res['OK']:
        return res
      ownerDN = res['Value'][0]

    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split( ';' )
      except AttributeError:
        pass

    for taskID in sorted( taskDict ):
      paramDict = taskDict[taskID]
      if paramDict['InputData']:
        transID = paramDict['TransformationID']

        oRequest = Request()
        transfer = Operation()
        transfer.Type = requestOperation
        transfer.TargetSE = paramDict['TargetSE']

        if isinstance( paramDict['InputData'], list ):
          files = paramDict['InputData']
        elif isinstance( paramDict['InputData'], basestring ):
          files = paramDict['InputData'].split( ';' )
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile( trFile )

        oRequest.addOperation( transfer )
        oRequest.RequestName = _requestName( transID, taskID )
        oRequest.OwnerDN = ownerDN
        oRequest.OwnerGroup = ownerGroup

      isValid = self.requestValidator.validate( oRequest )
      if not isValid['OK']:
        return isValid

      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )

  def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
    """ deal with a Request that has multiple operations

    :param transJson: list of lists of string and dictionaries, e.g.:

      .. code :: python

        body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
                 ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
               ]

    :param dict taskDict: dictionary of tasks, modified in this function
    :param str ownerDN: certificate DN used for the requests
    :param str onwerGroup: dirac group used for the requests

    :returns: None
    """
    failedTasks = []
    for taskID, task in taskDict.items():
      transID = task['TransformationID']
      if not task.get('InputData'):
        self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
        taskDict.pop(taskID)
        continue
      files = []

      oRequest = Request()
      if isinstance(task['InputData'], list):
        files = task['InputData']
      elif isinstance(task['InputData'], basestring):
        files = task['InputData'].split(';')

      # create the operations from the json structure
      for operationTuple in transJson:
        op = Operation()
        op.Type = operationTuple[0]
        for parameter, value in operationTuple[1].iteritems():
          setattr(op, parameter, value)

        for lfn in files:
          opFile = File()
          opFile.LFN = lfn
          op.addFile(opFile)

        oRequest.addOperation(op)

      result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
      if not result['OK']:
        failedTasks.append(taskID)
    # Remove failed tasks
    for taskID in failedTasks:
      taskDict.pop(taskID)

  def addRemovalRequests(self, lfnList):
    """Create removalRequests for lfns in lfnList and add it to the common request"""
    request = self._getRequestContainer()
    remove = Operation()
    remove.Type = "RemoveFile"

    for lfn in lfnList:
      rmFile = File()
      rmFile.LFN = lfn
      remove.addFile( rmFile )

    request.addOperation( remove )
    self.workflow_commons['Request'] = request

 def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
   """ Sets a removal request for a file including all replicas.
   """
   remove = Operation()
   remove.Type = "RemoveFile"
   if se:
     remove.TargetSE = se
   rmFile = File()
   rmFile.LFN = lfn
   if pfn:
     rmFile.PFN = pfn
   remove.addFile( rmFile )
   self.request.addOperation( remove )
   return S_OK()

  def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
    """ deal with a Request that has just one operation, as it was sofar

    :param transBody: string, can be an empty string
    :param dict taskDict: dictionary of tasks, modified in this function
    :param str ownerDN: certificate DN used for the requests
    :param str onwerGroup: dirac group used for the requests

    :returns: None
    """

    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split(';')
      except AttributeError:
        pass
    failedTasks = []
    # Do not remove sorted, we might pop elements in the loop
    for taskID, task in taskDict.iteritems():

      transID = task['TransformationID']

      oRequest = Request()
      transfer = Operation()
      transfer.Type = requestOperation
      transfer.TargetSE = task['TargetSE']

      # If there are input files
      if task.get('InputData'):
        if isinstance(task['InputData'], list):
          files = task['InputData']
        elif isinstance(task['InputData'], basestring):
          files = task['InputData'].split(';')
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile(trFile)

      oRequest.addOperation(transfer)
      result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
      if not result['OK']:
        failedTasks.append(taskID)
    # Remove failed tasks
    for taskID in failedTasks:
      taskDict.pop(taskID)

  def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
    """ Sets a registration request

    :param str lfn: LFN
    :param list se: list of SE
    :param list catalog: list of catalogs to use
    :param dict fileDict: file metadata
    """
    self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )

    for cat in catalog:

      register = Operation()
      register.Type = "RegisterFile"
      register.Catalog = cat
      register.TargetSE = targetSE

      regFile = File()
      regFile.LFN = lfn
      regFile.Checksum = fileDict.get( "Checksum", "" )
      regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
      regFile.Size = fileDict.get( "Size", 0 )
      regFile.GUID = fileDict.get( "GUID", "" )

      se = StorageElement( targetSE )
      pfn = se.getPfnForLfn( lfn )
      if not pfn["OK"]:
        self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
        return pfn
      regFile.PFN = pfn["Value"]

      register.addFile( regFile )
      self.request.addOperation( register )

    return S_OK()

 def test01TableDesc(self):
     """ table description """
     tableDict = RequestDB.getTableMeta()
     self.assertEqual("Request" in tableDict, True)
     self.assertEqual("Operation" in tableDict, True)
     self.assertEqual("File" in tableDict, True)
     self.assertEqual(tableDict["Request"], Request.tableDesc())
     self.assertEqual(tableDict["Operation"], Operation.tableDesc())
     self.assertEqual(tableDict["File"], File.tableDesc())

  def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
    """ Set removal request with the given credentials
    """
    oRequest = Request()
    oRequest.OwnerDN = ownerDN
    oRequest.OwnerGroup = ownerGroup
    oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
    oRequest.SourceComponent = 'JobCleaningAgent'

    removeFile = Operation()
    removeFile.Type = 'RemoveFile'

    removedFile = File()
    removedFile.LFN = lfn

    removeFile.addFile( removedFile )
    oRequest.addOperation( removeFile )

    return ReqClient().putRequest( oRequest )

  def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
    """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
    """
    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split( ';' )
      except AttributeError:
        pass

    for taskID in sorted( taskDict ):
      paramDict = taskDict[taskID]
      if paramDict['InputData']:
        transID = paramDict['TransformationID']

        oRequest = Request()
        transfer = Operation()
        transfer.Type = requestOperation
        transfer.TargetSE = paramDict['TargetSE']

        if type( paramDict['InputData'] ) == type( [] ):
          files = paramDict['InputData']
        elif type( paramDict['InputData'] ) == type( '' ):
          files = paramDict['InputData'].split( ';' )
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile( trFile )

        oRequest.addOperation( transfer )
        oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
        oRequest.OwnerDN = owner
        oRequest.OwnerGroup = ownerGroup

      isValid = gRequestValidator.validate( oRequest )
      if not isValid['OK']:
        return isValid

      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )

  def test__addMetadataToFiles( self ):
    resMeta = {'OK': True,
     'Value': {'Failed': {},
               'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
                                              'Checksum': '123456',
                                              'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
                                              'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
                                              'Mode': 436,
                                              'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
                                              'NumberOfLinks': 1,
                                              'Size': 5846023777,
                                              'Status': '-'},
                              '/lhcb/2.dst': {'ChecksumType': 'AD',
                                              'Checksum': '987654',
                                              'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
                                              'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
                                              'Mode': 436,
                                              'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
                                              'NumberOfLinks': 1,
                                              'Size': 5893396937,
                                              'Status': '-'}}}}

    self.rr.fc.getFileMetadata.return_value = resMeta

    file1 = File()
    file1.LFN = '/lhcb/1.dst'
    file2 = File()
    file2.LFN = '/lhcb/2.dst'

    toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
                  '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}

    res = self.rr._addMetadataToFiles( toSchedule )
    self.assertTrue(res['OK'])

    for lfn in toSchedule:
      self.assertEqual( res['Value'][lfn].LFN, lfn )
      for attr in ('GUID', 'Size', 'Checksum'):
        self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] )
      # AD should be transformed into Adler32
      self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )

  def __setReplicaRemovalRequest( self, lfn, se ):
    """ Sets a removal request for a replica.

    :param str lfn: LFN
    :param se:
    """
    if type( se ) == str:
      se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )

    removeReplica = Operation()

    removeReplica.Type = "RemoveReplica"
    removeReplica.TargetSE = se

    replicaToRemove = File()
    replicaToRemove.LFN = lfn

    removeReplica.addFile( replicaToRemove )

    self.request.addOperation( removeReplica )
    return S_OK()