Python File.PFN方法代码示例

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def __insertRegisterOperation( self, request, operation, toRegister ):
    """ add RegisterReplica operation

    :param Request request: request instance
    :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
    :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
    """
    log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )

    byTarget = {}
    for ftsFile in toRegister:
      if ftsFile.TargetSE not in byTarget:
        byTarget.setdefault( ftsFile.TargetSE, [] )
      byTarget[ftsFile.TargetSE].append( ftsFile )
    log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )

    for target, ftsFileList in byTarget.iteritems():
      log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
                                                                                            len( ftsFileList ) ) )
      registerOperation = Operation()
      registerOperation.Type = "RegisterReplica"
      registerOperation.Status = "Waiting"
      registerOperation.TargetSE = target
      targetSE = StorageElement( target )
      for ftsFile in ftsFileList:
        opFile = File()
        opFile.LFN = ftsFile.LFN
        pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
        if not pfn["OK"]:
          continue
        opFile.PFN = pfn["Value"]
        registerOperation.addFile( opFile )
      request.insertBefore( registerOperation, operation )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
    if self.getCSOption( "DelayedExternalDeletion", True ):
      gLogger.info( "Setting deletion request" )
      try:

        request = Request()
        request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
        physicalRemoval = Operation()
        physicalRemoval.Type = "PhysicalRemoval"
        physicalRemoval.TargetSE = SEName
        fileToRemove = File()
        fileToRemove.PFN = SEPFN
        physicalRemoval.addFile( fileToRemove )
        request.addOperation( physicalRemoval )
        return ReqClient().putRequest( request )
      except Exception as e:
        gLogger.exception( "Exception while setting deletion request" )
        return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
    else:
      gLogger.info( "Deleting external Sandbox" )
      try:
        return StorageElement( SEName ).removeFile( SEPFN )
      except Exception as e:
        gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
        return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
    """ Sets a registration request

    :param str lfn: LFN
    :param list se: list of SE
    :param list catalog: list of catalogs to use
    :param dict fileDict: file metadata
    """
    self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )

    for cat in catalog:

      register = Operation()
      register.Type = "RegisterFile"
      register.Catalog = cat
      register.TargetSE = targetSE

      regFile = File()
      regFile.LFN = lfn
      regFile.Checksum = fileDict.get( "Checksum", "" )
      regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
      regFile.Size = fileDict.get( "Size", 0 )
      regFile.GUID = fileDict.get( "GUID", "" )

      se = StorageElement( targetSE )
      pfn = se.getPfnForLfn( lfn )
      if not pfn["OK"]:
        self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
        return pfn
      regFile.PFN = pfn["Value"]

      register.addFile( regFile )
      self.request.addOperation( register )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):

    files = self.files( owner, group )

    putAndRegister = Operation()
    putAndRegister.Type = "PutAndRegister"
    putAndRegister.TargetSE = sourceSE
    for fname, lfn, size, checksum, guid in files:
      putFile = File()
      putFile.LFN = lfn
      putFile.PFN = fname
      putFile.Checksum = checksum
      putFile.ChecksumType = "adler32"
      putFile.Size = size
      putFile.GUID = guid
      putAndRegister.addFile( putFile )

    replicateAndRegister = Operation()
    replicateAndRegister.Type = "ReplicateAndRegister"
    replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
    for fname, lfn, size, checksum, guid in files:
      repFile = File()
      repFile.LFN = lfn
      repFile.Size = size
      repFile.Checksum = checksum
      repFile.ChecksumType = "adler32"
      replicateAndRegister.addFile( repFile )

    removeReplica = Operation()
    removeReplica.Type = "RemoveReplica"
    removeReplica.TargetSE = sourceSE
    for fname, lfn, size, checksum, guid in files:
      removeReplica.addFile( File( {"LFN": lfn } ) )

    removeFile = Operation()
    removeFile.Type = "RemoveFile"
    for fname, lfn, size, checksum, guid in files:
      removeFile.addFile( File( {"LFN": lfn } ) )

    removeFileInit = Operation()
    removeFileInit.Type = "RemoveFile"
    for fname, lfn, size, checksum, guid in files:
      removeFileInit.addFile( File( {"LFN": lfn } ) )

    req = Request()
    req.addOperation( removeFileInit )
    req.addOperation( putAndRegister )
    req.addOperation( replicateAndRegister )
    req.addOperation( removeReplica )
    req.addOperation( removeFile )
    return req

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
 def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
   """ Sets a removal request for a file including all replicas.
   """
   remove = Operation()
   remove.Type = "RemoveFile"
   if se:
     remove.TargetSE = se
   rmFile = File()
   rmFile.LFN = lfn
   if pfn:
     rmFile.PFN = pfn
   remove.addFile( rmFile )
   self.request.addOperation( remove )
   return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def getRegisterOperation( self, opFile, targetSE ):
    """ add RegisterReplica operation for file

    :param File opFile: operation file
    :param str targetSE: target SE
    """
    # # add RegisterReplica operation
    registerOperation = Operation()
    registerOperation.Type = "RegisterFile"
    registerOperation.TargetSE = targetSE

    registerFile = File()
    registerFile.LFN = opFile.LFN
    registerFile.PFN = opFile.PFN
    registerFile.GUID = opFile.GUID
    registerFile.Checksum = opFile.Checksum
    registerFile.ChecksumType = opFile.ChecksumType
    registerFile.Size = opFile.Size

    registerOperation.addFile( registerFile )
    return registerOperation

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def addRegisterReplica( self, opFile, targetSE ):
    """ add RegisterReplica operation for file

    :param File opFile: operation file
    :param str targetSE: target SE
    """
    # # add RegisterReplica operation
    registerOperation = Operation()
    registerOperation.Type = "RegisterFile"
    registerOperation.TargetSE = targetSE

    registerFile = File()
    registerFile.LFN = opFile.LFN
    registerFile.PFN = opFile.PFN
    registerFile.GUID = opFile.GUID
    registerFile.Checksum = opFile.Checksum
    registerFile.ChecksumType = opFile.ChecksumType
    registerFile.Size = opFile.Size

    registerOperation.addFile( registerFile )
    self.request.insertAfter( registerOperation, self.operation )
    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
    """ Sets a registration request

    :param str lfn: LFN
    :param list se: list of SE (or just string)
    :param list catalog: list (or string) of catalogs to use
    :param dict fileDict: file metadata
    """
    self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )

    if not isinstance( catalog, list ):
      catalog = [catalog]

    for cat in catalog:

      register = Operation()
      register.Type = "RegisterFile"
      register.Catalog = cat
      register.TargetSE = targetSE

      regFile = File()
      regFile.LFN = lfn
      regFile.Checksum = fileDict.get( "Checksum", "" )
      regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
      regFile.Size = fileDict.get( "Size", 0 )
      regFile.GUID = fileDict.get( "GUID", "" )

      se = StorageElement( targetSE )
      pfn = se.getURL( lfn, self.registrationProtocols )
      if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
        self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
        return pfn
      regFile.PFN = pfn["Value"]['Successful'][lfn]

      register.addFile( regFile )
      self.request.addOperation( register )

    return S_OK()

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def test02props( self ):
    """ test props and attributes  """
    theFile = File()

    # valid props
    theFile.FileID = 1
    self.assertEqual( theFile.FileID, 1 )
    theFile.Status = "Done"
    self.assertEqual( theFile.Status, "Done" )
    theFile.LFN = "/some/path/somewhere"
    self.assertEqual( theFile.LFN, "/some/path/somewhere" )
    theFile.PFN = "/some/path/somewhere"
    self.assertEqual( theFile.PFN, "/some/path/somewhere" )
    theFile.Attempt = 1
    self.assertEqual( theFile.Attempt, 1 )
    theFile.Size = 1
    self.assertEqual( theFile.Size, 1 )
    theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
    self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" )
    theFile.ChecksumType = "adler32"
    self.assertEqual( theFile.ChecksumType, "ADLER32" )
    theFile.Checksum = "123456"
    self.assertEqual( theFile.Checksum, "123456" )

    # #
    theFile.Checksum = None
    theFile.ChecksumType = None
    self.assertEqual( theFile.Checksum, "" )
    self.assertEqual( theFile.ChecksumType, "" )

    # # invalid props

    # FileID
    try:
      theFile.FileID = "foo"
    except Exception, error:
      self.assertEqual( isinstance( error, ValueError ), True )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
  def test05FTS( self ):
    """ FTS state machine """

    req = Request()
    req.RequestName = "FTSTest"

    ftsTransfer = Operation()
    ftsTransfer.Type = "ReplicateAndRegister"
    ftsTransfer.TargetSE = "CERN-USER"

    ftsFile = File()
    ftsFile.LFN = "/a/b/c"
    ftsFile.Checksum = "123456"
    ftsFile.ChecksumType = "Adler32"

    ftsTransfer.addFile( ftsFile )
    req.addOperation( ftsTransfer )

    self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status )
    self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status )

    # # scheduled
    ftsFile.Status = "Scheduled"

    self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status )
    self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status )

    # # add new operation before FTS
    insertBefore = Operation()
    insertBefore.Type = "RegisterReplica"
    insertBefore.TargetSE = "CERN-USER"
    insertFile = File()
    insertFile.LFN = "/a/b/c"
    insertFile.PFN = "http://foo/bar"
    insertBefore.addFile( insertFile )
    req.insertBefore( insertBefore, ftsTransfer )

    self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status )
    self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status )
    self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status )

    # # prev done
    insertFile.Status = "Done"

    self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status )
    self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status )
    self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status )

    # # reschedule
    ftsFile.Status = "Waiting"

    self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
    self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
    self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status )

    # # fts done
    ftsFile.Status = "Done"

    self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
    self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
    self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )

# 需要导入模块: from DIRAC.RequestManagementSystem.Client.File import File [as 别名]
# 或者: from DIRAC.RequestManagementSystem.Client.File.File import PFN [as 别名]
    gLogger.error( "%s is not a file" % PFN )
    DIRAC.exit( -1 )

  PFN = os.path.abspath( PFN )
  size = os.path.getsize( PFN )
  adler32 = fileAdler( PFN )

  request = Request()
  request.RequestName = requestName

  putAndRegister = Operation()
  putAndRegister.Type = "PutAndRegister"
  putAndRegister.TargetSE = targetSE
  opFile = File()
  opFile.LFN = LFN
  opFile.PFN = PFN
  opFile.Size = size
  opFile.Checksum = adler32
  opFile.ChecksumType = "ADLER32"
  putAndRegister.addFile( opFile )
  request.addOperation( putAndRegister )
  reqClient = ReqClient()
  putRequest = reqClient.putRequest( request )
  if not putRequest["OK"]:
    gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) )
    DIRAC.exit( -1 )

  gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName )
  gLogger.always( "You can monitor its status using command: 'dirac-rms-show-request %s'" % requestName )
  DIRAC.exit( 0 )