Python DataManager.getActiveReplicas方法代码示例

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def getFile(lfn, se=""):
    dm = DataManager()

    download_ok = 0
    get_active_replicas_ok = False
    lfn_on_se = False
    error_msg = ""
    if se:
        for i in range(0, 5):
            result = dm.getActiveReplicas(lfn)
            if result["OK"] and result["Value"]["Successful"]:
                get_active_replicas_ok = True
                lfnReplicas = result["Value"]["Successful"]
                if se in lfnReplicas[lfn]:
                    lfn_on_se = True
                    break
            time.sleep(3)
            print "- Get replicas for %s failed, try again" % lfn

        if not get_active_replicas_ok:
            return S_ERROR("Get replicas error: %s" % lfn)

    if lfn_on_se:
        se = StorageElement(se)
        # try 5 times
        for j in range(0, 5):
            result = se.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- %s getStorageFile(%s) failed, try again" % (lfn, se)
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 1
            else:
                error_msg = "Downloading %s from SE %s error!" % (lfn, se)
        else:
            error_msg = result["Message"]
    else:
        if se:
            print 'File %s not found on SE "%s" after %s tries, trying other SE' % (lfn, se, i + 1)
        # try 5 times
        for j in range(0, 5):
            result = dm.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- getFile(%s) failed, try again" % lfn
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 2
            else:
                error_msg = "Downloading %s from random SE error!" % lfn
        else:
            error_msg = result["Message"]

    if download_ok:
        return S_OK({lfn: {"DownloadOK": download_ok, "Retry": j + 1}})

    return S_ERROR(error_msg)

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def doesFileExist(lfn):
    from DIRAC.DataManagementSystem.Client.DataManager import DataManager
    dm = DataManager()
    result = dm.getActiveReplicas(lfn)
    if result[('Value')][('Successful')]:
        return True
    else:
        return False

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def doesFileExist(lfn):
    from DIRAC.DataManagementSystem.Client.DataManager import DataManager
    dm = DataManager()
    result = dm.getActiveReplicas(lfn)
    if not result['OK']:
        print "ERROR",result['Message']
        return False
    if lfn in result['Value']['Successful']:
        return True
    else:
        return False

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def getFilesToStage( lfnList ):
  """ Utility that returns out of a list of LFNs those files that are offline,
      and those for which at least one copy is online
  """
  onlineLFNs = set()
  offlineLFNsDict = {}

  if not lfnList:
    return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
  
  dm = DataManager()

  lfnListReplicas = dm.getActiveReplicas( lfnList )
  if not lfnListReplicas['OK']:
    return lfnListReplicas

  seObjectsDict = dict()
  seToLFNs = dict()
  
  if lfnListReplicas['Value']['Failed']:
    return S_ERROR( "Failures in getting replicas" )
  for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
    for se, _ in ld.iteritems():
      seObjectsDict.setdefault( se, StorageElement( se ) )
      seToLFNs.setdefault( se, list() ).append( lfn )

  for se, lfnsInSEList in seToLFNs.iteritems():
    fileMetadata = seObjectsDict[se].getFileMetadata( lfnsInSEList )
    if not fileMetadata['OK']:
      return fileMetadata

    if fileMetadata['Value']['Failed']:
      return S_ERROR( "Failures in getting file metadata" )
    # is there at least one online?
    for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
      if mDict['Cached']:
        onlineLFNs.add( lfn )

  offlineLFNs = set( lfnList ).difference( onlineLFNs )
  

  for offlineLFN in offlineLFNs:
    ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
    random.shuffle( ses )
    se = ses[0]
    offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
  
  return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]

#.........这里部分代码省略.........
        gLogger.error( "RequestPreparation.prepareNewReplicas: Failed to update replica failures.", res['Message'] )
    if replicaMetadata:
      gLogger.info( "RequestPreparation.prepareNewReplicas: %s replica metadata to be updated." % len( replicaMetadata ) )
      # Sets the Status='Waiting' of CacheReplicas records that are OK with catalogue checks
      res = self.stagerClient.updateReplicaInformation( replicaMetadata )
      if not res['OK']:
        gLogger.error( "RequestPreparation.prepareNewReplicas: Failed to update replica metadata.", res['Message'] )
    return S_OK()

  def __getNewReplicas( self ):
    """ This obtains the New replicas from the Replicas table and for each LFN the requested storage element """
    # First obtain the New replicas from the CacheReplicas table
    res = self.stagerClient.getCacheReplicas( {'Status':'New'} )
    if not res['OK']:
      gLogger.error( "RequestPreparation.__getNewReplicas: Failed to get replicas with New status.", res['Message'] )
      return res
    if not res['Value']:
      gLogger.debug( "RequestPreparation.__getNewReplicas: No New replicas found to process." )
      return S_OK()
    else:
      gLogger.debug( "RequestPreparation.__getNewReplicas: Obtained %s New replicas(s) to process." % len( res['Value'] ) )
    replicas = {}
    replicaIDs = {}
    for replicaID, info in res['Value'].items():
      lfn = info['LFN']
      storageElement = info['SE']
      replicas.setdefault( lfn, {} )[storageElement] = replicaID
      replicaIDs[replicaID] = ( lfn, storageElement )
    return S_OK( {'Replicas':replicas, 'ReplicaIDs':replicaIDs} )

  def __getExistingFiles( self, lfns ):
    """ This checks that the files exist in the FileCatalog. """
    res = self.fileCatalog.exists( list( set( lfns ) ) )
    if not res['OK']:
      gLogger.error( "RequestPreparation.__getExistingFiles: Failed to determine whether files exist.", res['Message'] )
      return res
    failed = res['Value']['Failed']
    success = res['Value']['Successful']
    exist = [lfn for lfn, exists in success.items() if exists]
    missing = list( set( success ) - set( exist ) )
    if missing:
      reason = 'LFN not registered in the FC'
      gLogger.warn( "RequestPreparation.__getExistingFiles: %s" % reason, '\n'.join( [''] + missing ) )
      self.__reportProblematicFiles( missing, 'LFN-LFC-DoesntExist' )
      missing = dict.fromkeys( missing, reason )
    else:
      missing = {}
    return S_OK( {'Exist':exist, 'Missing':missing, 'Failed':failed} )

  def __getFileSize( self, lfns ):
    """ This obtains the file size from the FileCatalog. """
    fileSizes = {}
    zeroSize = {}
    res = self.fileCatalog.getFileSize( lfns )
    if not res['OK']:
      gLogger.error( "RequestPreparation.__getFileSize: Failed to get sizes for files.", res['Message'] )
      return res
    failed = res['Value']['Failed']
    for lfn, size in res['Value']['Successful'].items():
      if size == 0:
        zeroSize[lfn] = "LFN registered with zero size in the FileCatalog"
      else:
        fileSizes[lfn] = size
    if zeroSize:
      for lfn, reason in zeroSize.items():
        gLogger.warn( "RequestPreparation.__getFileSize: %s" % reason, lfn )
      self.__reportProblematicFiles( zeroSize.keys(), 'LFN-LFC-ZeroSize' )
    return S_OK( {'FileSizes':fileSizes, 'ZeroSize':zeroSize, 'Failed':failed} )

  def __getFileReplicas( self, lfns ):
    """ This obtains the replicas from the FileCatalog. """
    replicas = {}
    noReplicas = {}
    res = self.dm.getActiveReplicas( lfns )
    if not res['OK']:
      gLogger.error( "RequestPreparation.__getFileReplicas: Failed to obtain file replicas.", res['Message'] )
      return res
    failed = res['Value']['Failed']
    for lfn, lfnReplicas in res['Value']['Successful'].items():
      if len( lfnReplicas.keys() ) == 0:
        noReplicas[lfn] = "LFN registered with zero replicas in the FileCatalog"
      else:
        replicas[lfn] = lfnReplicas
    if noReplicas:
      for lfn, reason in noReplicas.items():
        gLogger.warn( "RequestPreparation.__getFileReplicas: %s" % reason, lfn )
      self.__reportProblematicFiles( noReplicas.keys(), 'LFN-LFC-NoReplicas' )
    return S_OK( {'Replicas':replicas, 'ZeroReplicas':noReplicas, 'Failed':failed} )

  def __reportProblematicFiles( self, lfns, reason ):
    return S_OK()
    res = self.dataIntegrityClient.setFileProblematic( lfns, reason, sourceComponent = 'RequestPreparationAgent' )
    if not res['OK']:
      gLogger.error( "RequestPreparation.__reportProblematicFiles: Failed to report missing files.", res['Message'] )
      return res
    if res['Value']['Successful']:
      gLogger.info( "RequestPreparation.__reportProblematicFiles: Successfully reported %s missing files." % len( res['Value']['Successful'] ) )
    if res['Value']['Failed']:
      gLogger.info( "RequestPreparation.__reportProblematicFiles: Failed to report %s problematic files." % len( res['Value']['Failed'] ) )
    return res

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]

#.........这里部分代码省略.........
    """
    return self.ftsManager.deleteFTSFiles( operationID, opFileIDList )

  def ftsSchedule( self, requestID, operationID, opFileList ):
    """ schedule lfn for FTS job

    :param int requestID: RequestDB.Request.RequestID
    :param int operationID: RequestDB.Operation.OperationID
    :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
    """

    fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ]
    res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
    if not res['OK']:
      self.log.error( "ftsSchedule: %s" % res['Message'] )
      return S_ERROR( "ftsSchedule: %s" % res['Message'] )

    ftsFiles = []

    # # this will be returned on success
    result = { "Successful": [], "Failed": {} }

    for fileJSON, sourceSEs, targetSEs in opFileList:

      lfn = fileJSON.get( "LFN", "" )
      size = int( fileJSON.get( "Size", 0 ) )
      fileID = int( fileJSON.get( "FileID", 0 ) )
      opID = int( fileJSON.get( "OperationID", 0 ) )

      self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
                                                                                                 sourceSEs,
                                                                                                 targetSEs ) )

      res = self.dataManager.getActiveReplicas( lfn )
      if not res['OK']:
        self.log.error( "ftsSchedule: %s" % res['Message'] )
        result["Failed"][fileID] = res['Message']
        continue
      replicaDict = res['Value']

      if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
        result["Failed"][fileID] = "no active replicas found"
        continue
      replicaDict = replicaDict["Successful"].get( lfn, {} )
      # # use valid replicas only
      validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )

      if not validReplicasDict:
        self.log.warn( "No active replicas found in sources" )
        result["Failed"][fileID] = "no active replicas found in sources"
        continue

      tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
      if not tree['OK']:
        self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
        result["Failed"][fileID] = tree['Message']
        continue
      tree = tree['Value']

      self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )

      for repDict in tree.values():
        self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
                                                                                repDict["Ancestor"],
                                                                                repDict["SourceSE"],
                                                                                repDict["TargetSE"] ) )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def filterReplicas( opFile, logger = None, dataManager = None, seCache = None ):
  """ filter out banned/invalid source SEs """
  from DIRAC.Core.Utilities.Adler import compareAdler

  if not logger:
    logger = gLogger
  if not dataManager:
    dataManager = DataManager()
  if not seCache:
    seCache = {}

  log = logger.getSubLogger( "filterReplicas" )
  ret = { "Valid" : [], "Banned" : [], "Bad" : [], 'NoReplicas':[], 'NoPFN':[] }

  replicas = dataManager.getActiveReplicas( opFile.LFN )
  if not replicas["OK"]:
    log.error( replicas["Message"] )
    return replicas
  reNotExists = re.compile( "not such file or directory" )
  replicas = replicas["Value"]
  failed = replicas["Failed"].get( opFile.LFN , "" )
  if reNotExists.match( failed.lower() ):
    opFile.Status = "Failed"
    opFile.Error = failed
    return S_ERROR( failed )

  replicas = replicas["Successful"].get( opFile.LFN, {} )

  for repSEName in replicas:

    repSE = seCache[repSEName] if repSEName in seCache else \
            seCache.setdefault( repSEName, StorageElement( repSEName ) )

    pfn = repSE.getPfnForLfn( opFile.LFN )
    if not pfn["OK"] or opFile.LFN not in pfn['Value']['Successful']:
      log.warn( "unable to create pfn for %s lfn at %s: %s" % ( opFile.LFN, repSEName, pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( opFile.LFN ) ) ) )
      ret["NoPFN"].append( repSEName )
    else:
      pfn = pfn["Value"]['Successful'][ opFile.LFN ]

      repSEMetadata = repSE.getFileMetadata( pfn )
      error = repSEMetadata.get( 'Message', repSEMetadata.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
      if error:
        log.warn( 'unable to get metadata at %s for %s' % ( repSEName, opFile.LFN ), error )
        if 'File does not exist' in error:
          ret['NoReplicas'].append( repSEName )
        else:
          log.verbose( "StorageElement '%s' is banned for reading" % ( repSEName ) )
          ret["Banned"].append( repSEName )
      else:
        repSEMetadata = repSEMetadata['Value']['Successful'][pfn]

        seChecksum = repSEMetadata.get( "Checksum" )
        if opFile.Checksum and seChecksum and not compareAdler( seChecksum, opFile.Checksum ) :
          log.warn( " %s checksum mismatch: %s %s:%s" % ( opFile.LFN,
                                                               opFile.Checksum,
                                                               repSE,
                                                               seChecksum ) )
          ret["Bad"].append( repSEName )
        else:
          # # if we're here repSE is OK
          ret["Valid"].append( repSEName )

  return S_OK( ret )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def filterReplicas(opFile, logger=None, dataManager=None):
  """ filter out banned/invalid source SEs """

  if logger is None:
    logger = gLogger
  if dataManager is None:
    dataManager = DataManager()

  log = logger.getSubLogger("filterReplicas")
  result = defaultdict(list)

  replicas = dataManager.getActiveReplicas(opFile.LFN, getUrl=False)
  if not replicas["OK"]:
    log.error('Failed to get active replicas', replicas["Message"])
    return replicas
  reNotExists = re.compile(r".*such file.*")
  replicas = replicas["Value"]
  failed = replicas["Failed"].get(opFile.LFN, "")
  if reNotExists.match(failed.lower()):
    opFile.Status = "Failed"
    opFile.Error = failed
    return S_ERROR(failed)

  replicas = replicas["Successful"].get(opFile.LFN, {})
  noReplicas = False
  if not replicas:
    allReplicas = dataManager.getReplicas(opFile.LFN, getUrl=False)
    if allReplicas['OK']:
      allReplicas = allReplicas['Value']['Successful'].get(opFile.LFN, {})
      if not allReplicas:
        result['NoReplicas'].append(None)
        noReplicas = True
      else:
        # There are replicas but we cannot get metadata because the replica is not active
        result['NoActiveReplicas'] += list(allReplicas)
      log.verbose("File has no%s replica in File Catalog" % ('' if noReplicas else ' active'), opFile.LFN)
    else:
      return allReplicas

  if not opFile.Checksum or hexAdlerToInt(opFile.Checksum) is False:
    # Set Checksum to FC checksum if not set in the request
    fcMetadata = FileCatalog().getFileMetadata(opFile.LFN)
    fcChecksum = fcMetadata.get(
        'Value',
        {}).get(
        'Successful',
        {}).get(
        opFile.LFN,
        {}).get('Checksum')
    # Replace opFile.Checksum if it doesn't match a valid FC checksum
    if fcChecksum:
      if hexAdlerToInt(fcChecksum) is not False:
        opFile.Checksum = fcChecksum
        opFile.ChecksumType = fcMetadata['Value']['Successful'][opFile.LFN].get('ChecksumType', 'Adler32')
      else:
        opFile.Checksum = None

  # If no replica was found, return what we collected as information
  if not replicas:
    return S_OK(result)

  for repSEName in replicas:
    repSEMetadata = StorageElement(repSEName).getFileMetadata(opFile.LFN)
    error = repSEMetadata.get('Message', repSEMetadata.get('Value', {}).get('Failed', {}).get(opFile.LFN))
    if error:
      log.warn('unable to get metadata at %s for %s' % (repSEName, opFile.LFN), error.replace('\n', ''))
      if 'File does not exist' in error:
        result['NoReplicas'].append(repSEName)
      else:
        result["NoMetadata"].append(repSEName)
    elif not noReplicas:
      repSEMetadata = repSEMetadata['Value']['Successful'][opFile.LFN]

      seChecksum = hexAdlerToInt(repSEMetadata.get("Checksum"))
      # As from here seChecksum is an integer or False, not a hex string!
      if seChecksum is False and opFile.Checksum:
        result['NoMetadata'].append(repSEName)
      elif not seChecksum and opFile.Checksum:
        opFile.Checksum = None
        opFile.ChecksumType = None
      elif seChecksum and (not opFile.Checksum or opFile.Checksum == 'False'):
        # Use the SE checksum (convert to hex) and force type to be Adler32
        opFile.Checksum = intAdlerToHex(seChecksum)
        opFile.ChecksumType = 'Adler32'
      if not opFile.Checksum or not seChecksum or compareAdler(
              intAdlerToHex(seChecksum), opFile.Checksum):
        # # All checksums are OK
        result["Valid"].append(repSEName)
      else:
        log.warn(" %s checksum mismatch, FC: '%s' @%s: '%s'" %
                 (opFile.LFN, opFile.Checksum, repSEName, intAdlerToHex(seChecksum)))
        result["Bad"].append(repSEName)
    else:
      # If a replica was found somewhere, don't set the file as no replicas
      result['NoReplicas'] = []

  return S_OK(result)

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
class InputDataAgent( OptimizerModule ):
  """
      The specific Optimizer must provide the following methods:
      - initializeOptimizer() before each execution cycle
      - checkJob() - the main method called for each job
  """

  #############################################################################
  def initializeOptimizer( self ):
    """Initialize specific parameters for JobSanityAgent.
    """
    self.failedMinorStatus = self.am_getOption( '/FailedJobStatus', 'Input Data Not Available' )
    #this will ignore failover SE files
    self.checkFileMetadata = self.am_getOption( 'CheckFileMetadata', True )

    self.dataManager = DataManager()
    self.resourceStatus = ResourceStatus()
    self.fc = FileCatalog()

    self.seToSiteMapping = {}
    self.lastCScheck = 0
    self.cacheLength = 600

    return S_OK()

  #############################################################################
  def checkJob( self, job, classAdJob ):
    """
    This method does the optimization corresponding to this Agent, 
    it is call for each job by the Optimizer framework
    """

    result = self.jobDB.getInputData( job )
    if not result['OK']:
      self.log.warn( 'Failed to get input data from JobdB for %s' % ( job ) )
      self.log.warn( result['Message'] )
      return result
    if not result['Value']:
      self.log.verbose( 'Job %s has no input data requirement' % ( job ) )
      return self.setNextOptimizer( job )

    #Check if we already executed this Optimizer and the input data is resolved
    res = self.getOptimizerJobInfo( job, self.am_getModuleParam( 'optimizerName' ) )
    if res['OK'] and len( res['Value'] ):
      pass
    else:
      self.log.verbose( 'Job %s has an input data requirement and will be processed' % ( job ) )
      inputData = result['Value']
      result = self.__resolveInputData( job, inputData )
      if not result['OK']:
        self.log.warn( result['Message'] )
        return result

    return self.setNextOptimizer( job )

  #############################################################################
  def __resolveInputData( self, job, inputData ):
    """This method checks the file catalog for replica information.
    """
    lfns = [ fname.replace( 'LFN:', '' ) for fname in inputData ]

    start = time.time()
    # In order to place jobs on Hold if a certain SE is banned we need first to check first if
    # if the replicas are really available
    replicas = self.dataManager.getActiveReplicas( lfns )
    timing = time.time() - start
    self.log.verbose( 'Catalog Replicas Lookup Time: %.2f seconds ' % ( timing ) )
    if not replicas['OK']:
      self.log.warn( replicas['Message'] )
      return replicas

    replicaDict = replicas['Value']

    siteCandidates = self.__checkReplicas( job, replicaDict )

    if not siteCandidates['OK']:
      self.log.warn( siteCandidates['Message'] )
      return siteCandidates

    if self.checkFileMetadata:
      guids = True
      start = time.time()
      guidDict = self.fc.getFileMetadata( lfns )
      timing = time.time() - start
      self.log.info( 'Catalog Metadata Lookup Time: %.2f seconds ' % ( timing ) )

      if not guidDict['OK']:
        self.log.warn( guidDict['Message'] )
        guids = False

      failed = guidDict['Value']['Failed']
      if failed:
        self.log.warn( 'Failed to establish some GUIDs' )
        self.log.warn( failed )
        guids = False

      if guids:
        for lfn, reps in replicaDict['Successful'].items():
          guidDict['Value']['Successful'][lfn].update( reps )
        replicas = guidDict
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def filterReplicas( opFile, logger = None, dataManager = None ):
  """ filter out banned/invalid source SEs """

  if logger is None:
    logger = gLogger
  if dataManager is None:
    dataManager = DataManager()

  log = logger.getSubLogger( "filterReplicas" )
  ret = { "Valid" : [], "NoMetadata" : [], "Bad" : [], 'NoReplicas':[], 'NoPFN':[] }

  replicas = dataManager.getActiveReplicas( opFile.LFN )
  if not replicas["OK"]:
    log.error( 'Failed to get active replicas', replicas["Message"] )
    return replicas
  reNotExists = re.compile( r".*such file.*" )
  replicas = replicas["Value"]
  failed = replicas["Failed"].get( opFile.LFN , "" )
  if reNotExists.match( failed.lower() ):
    opFile.Status = "Failed"
    opFile.Error = failed
    return S_ERROR( failed )

  replicas = replicas["Successful"].get( opFile.LFN, {} )

  if not opFile.Checksum:
    # Set Checksum to FC checksum if not set in the request
    fcMetadata = FileCatalog().getFileMetadata( opFile.LFN )
    fcChecksum = fcMetadata.get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN, {} ).get( 'Checksum', '' )
    # Replace opFile.Checksum if it doesn't match a valid FC checksum
    if fcChecksum:
      opFile.Checksum = fcChecksum
      opFile.ChecksumType = fcMetadata['Value']['Successful'][opFile.LFN].get( 'ChecksumType', 'Adler32' )

  for repSEName in replicas:

    repSE = StorageElement( repSEName )


    repSEMetadata = repSE.getFileMetadata( opFile.LFN )
    error = repSEMetadata.get( 'Message', repSEMetadata.get( 'Value', {} ).get( 'Failed', {} ).get( opFile.LFN ) )
    if error:
      log.warn( 'unable to get metadata at %s for %s' % ( repSEName, opFile.LFN ), error.replace( '\n', '' ) )
      if 'File does not exist' in error:
        ret['NoReplicas'].append( repSEName )
      else:
        ret["NoMetadata"].append( repSEName )
    else:
      repSEMetadata = repSEMetadata['Value']['Successful'][opFile.LFN]

      seChecksum = repSEMetadata.get( "Checksum" )
      if ( opFile.Checksum and seChecksum and compareAdler( seChecksum, opFile.Checksum ) ) or\
         ( not opFile.Checksum and not seChecksum ):
        # # All checksums are OK
        ret["Valid"].append( repSEName )
      else:
        log.warn( " %s checksum mismatch, FC: '%s' @%s: '%s'" % ( opFile.LFN,
                                                              opFile.Checksum,
                                                              repSEName,
                                                              seChecksum ) )
        ret["Bad"].append( repSEName )

  return S_OK( ret )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def filterReplicas( opFile, logger = None, dataManager = None, seCache = None ):
  """ filter out banned/invalid source SEs """

  if not logger:
    logger = gLogger
  if not dataManager:
    dataManager = DataManager()
  if not seCache:
    seCache = {}

  log = logger.getSubLogger( "filterReplicas" )
  ret = { "Valid" : [], "NoMetadata" : [], "Bad" : [], 'NoReplicas':[], 'NoPFN':[] }

  replicas = dataManager.getActiveReplicas( opFile.LFN )
  if not replicas["OK"]:
    log.error( replicas["Message"] )
    return replicas
  reNotExists = re.compile( r".*such file.*" )
  replicas = replicas["Value"]
  failed = replicas["Failed"].get( opFile.LFN , "" )
  if reNotExists.match( failed.lower() ):
    opFile.Status = "Failed"
    opFile.Error = failed
    return S_ERROR( failed )

  replicas = replicas["Successful"].get( opFile.LFN, {} )

  for repSEName in replicas:

    repSE = seCache[repSEName] if repSEName in seCache else \
            seCache.setdefault( repSEName, StorageElement( repSEName ) )

    pfn = repSE.getPfnForLfn( opFile.LFN )
    if not pfn["OK"] or opFile.LFN not in pfn['Value']['Successful']:
      log.warn( "unable to create pfn for %s lfn at %s: %s" % ( opFile.LFN,
                                                                repSEName,
                                                                pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( opFile.LFN ) ) ) )
      ret["NoPFN"].append( repSEName )
    else:
      pfn = pfn["Value"]['Successful'][ opFile.LFN ]

      repSEMetadata = repSE.getFileMetadata( pfn )
      error = repSEMetadata.get( 'Message', repSEMetadata.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
      if error:
        log.warn( 'unable to get metadata at %s for %s' % ( repSEName, opFile.LFN ), error.replace( '\n', '' ) )
        if 'File does not exist' in error:
          ret['NoReplicas'].append( repSEName )
        else:
          ret["NoMetadata"].append( repSEName )
      else:
        repSEMetadata = repSEMetadata['Value']['Successful'][pfn]

        seChecksum = repSEMetadata.get( "Checksum" )
        if opFile.Checksum and seChecksum and not compareAdler( seChecksum, opFile.Checksum ) :
          # The checksum in the request may be wrong, check with FC
          fcMetadata = FileCatalog().getFileMetadata( opFile.LFN )
          fcChecksum = fcMetadata.get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN, {} ).get( 'Checksum' )
          if fcChecksum and fcChecksum != opFile.Checksum and compareAdler( fcChecksum , seChecksum ):
            opFile.Checksum = fcChecksum
            ret['Valid'].append( repSEName )
          else:
            log.warn( " %s checksum mismatch, request: %s @%s: %s" % ( opFile.LFN,
                                                                       opFile.Checksum,
                                                                       repSEName,
                                                                       seChecksum ) )
            ret["Bad"].append( repSEName )
        else:
          # # if we're here repSE is OK
          ret["Valid"].append( repSEName )

  return S_OK( ret )

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getActiveReplicas [as 别名]
def filterReplicas(opFile, logger=None, dataManager=None):
    """ filter out banned/invalid source SEs """

    if logger is None:
        logger = gLogger
    if dataManager is None:
        dataManager = DataManager()

    log = logger.getSubLogger("filterReplicas")
    ret = {"Valid": [], "NoMetadata": [], "Bad": [], "NoReplicas": [], "NoPFN": []}

    replicas = dataManager.getActiveReplicas(opFile.LFN)
    if not replicas["OK"]:
        log.error("Failed to get active replicas", replicas["Message"])
        return replicas
    reNotExists = re.compile(r".*such file.*")
    replicas = replicas["Value"]
    failed = replicas["Failed"].get(opFile.LFN, "")
    if reNotExists.match(failed.lower()):
        opFile.Status = "Failed"
        opFile.Error = failed
        return S_ERROR(failed)

    replicas = replicas["Successful"].get(opFile.LFN, {})
    noReplicas = False
    if not replicas:
        allReplicas = dataManager.getReplicas(opFile.LFN)
        if allReplicas["OK"]:
            allReplicas = allReplicas["Value"]["Successful"].get(opFile.LFN, {})
            if not allReplicas:
                ret["NoReplicas"].append(None)
                noReplicas = True
            else:
                # We try inactive replicas to see if maybe the file doesn't exist at all
                replicas = allReplicas
            log.warn("File has no%s replica in File Catalog" % ("" if noReplicas else " active"), opFile.LFN)
        else:
            return allReplicas

    if not opFile.Checksum:
        # Set Checksum to FC checksum if not set in the request
        fcMetadata = FileCatalog().getFileMetadata(opFile.LFN)
        fcChecksum = fcMetadata.get("Value", {}).get("Successful", {}).get(opFile.LFN, {}).get("Checksum")
        # Replace opFile.Checksum if it doesn't match a valid FC checksum
        if fcChecksum:
            opFile.Checksum = fcChecksum
            opFile.ChecksumType = fcMetadata["Value"]["Successful"][opFile.LFN].get("ChecksumType", "Adler32")

    for repSEName in replicas:
        repSEMetadata = StorageElement(repSEName).getFileMetadata(opFile.LFN)
        error = repSEMetadata.get("Message", repSEMetadata.get("Value", {}).get("Failed", {}).get(opFile.LFN))
        if error:
            log.warn("unable to get metadata at %s for %s" % (repSEName, opFile.LFN), error.replace("\n", ""))
            if "File does not exist" in error:
                ret["NoReplicas"].append(repSEName)
            else:
                ret["NoMetadata"].append(repSEName)
        elif not noReplicas:
            repSEMetadata = repSEMetadata["Value"]["Successful"][opFile.LFN]

            seChecksum = repSEMetadata.get("Checksum")
            if not seChecksum and opFile.Checksum:
                opFile.Checksum = None
                opFile.ChecksumType = None
            elif seChecksum and not opFile.Checksum:
                opFile.Checksum = seChecksum
            if not opFile.Checksum or not seChecksum or compareAdler(seChecksum, opFile.Checksum):
                # # All checksums are OK
                ret["Valid"].append(repSEName)
            else:
                log.warn(
                    " %s checksum mismatch, FC: '%s' @%s: '%s'" % (opFile.LFN, opFile.Checksum, repSEName, seChecksum)
                )
                ret["Bad"].append(repSEName)
        else:
            # If a replica was found somewhere, don't set the file as no replicas
            ret["NoReplicas"] = []

    return S_OK(ret)