Python ResourceStatus.getStorageElementStatus方法代码示例

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........
      return S_ERROR( errStr )
    if referenceType in res['Value']:
      configPath = cfgPath( self.rootConfigPath, storageName, referenceType )
      referenceName = gConfig.getValue( configPath )
      result = self._getConfigStorageName( referenceName, 'Alias' )
      if not result['OK']:
        return result
      resolvedName = result['Value']
    else:
      resolvedName = storageName
    return S_OK( resolvedName )

  def _getConfigStorageOptions( self, storageName ):
    """ Get the options associated to the StorageElement as defined in the CS
    """
    storageConfigPath = cfgPath( self.rootConfigPath, storageName )
    res = gConfig.getOptions( storageConfigPath )
    if not res['OK']:
      errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )
    options = res['Value']
    optionsDict = {}
    for option in options:

      if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']:
        continue
      optionConfigPath = cfgPath( storageConfigPath, option )
      if option in [ 'VO' ]:
        optionsDict[option] = gConfig.getValue( optionConfigPath, [] )
      else:
        optionsDict[option] = gConfig.getValue( optionConfigPath, '' )

    res = self.resourceStatus.getStorageElementStatus( storageName )
    if not res[ 'OK' ]:
      errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )

    # For safety, we did not add the ${statusType}Access keys
    # this requires modifications in the StorageElement class

    # We add the dictionary with the statusTypes and values
    # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
    optionsDict.update( res[ 'Value' ][ storageName ] )

    return S_OK( optionsDict )

  def _getConfigStorageProtocols( self, storageName ):
    """ Protocol specific information is present as sections in the Storage configuration
    """
    storageConfigPath = cfgPath( self.rootConfigPath, storageName )
    res = gConfig.getSections( storageConfigPath )
    if not res['OK']:
      errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections"
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )
    protocolSections = res['Value']
    sortedProtocolSections = sortList( protocolSections )
    protocolDetails = []
    for protocolSection in sortedProtocolSections:
      res = self._getConfigStorageProtocolDetails( storageName, protocolSection )
      if not res['OK']:
        return res
      protocolDetails.append( res['Value'] )
    return S_OK( protocolDetails )

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]
storageCFGBase = "/Resources/StorageElements"

res = gConfig.getSections( storageCFGBase, True )
if not res[ 'OK' ]:
  gLogger.error( 'Failed to get storage element info' )
  gLogger.error( res[ 'Message' ] )
  DIRAC.exit( -1 )
  
gLogger.info( "%s %s %s" % ( 'Storage Element'.ljust( 25 ), 'Read Status'.rjust( 15 ), 'Write Status'.rjust( 15 ) ) )

seList = sortList( res[ 'Value' ] )

resourceStatus = ResourceStatus()
 
res    = resourceStatus.getStorageElementStatus( seList )
if not res[ 'OK' ]:
  gLogger.error( "Failed to get StorageElement status for %s" % str( seList ) )

for k,v in res[ 'Value' ].items():
  
  readState, writeState = 'Active', 'Active'
  
  if v.has_key( 'Read' ):
    readState = v[ 'Read' ]  
  
  if v.has_key( 'Write' ):
    writeState = v[ 'Write']
  gLogger.notice("%s %s %s" % ( k.ljust(25),readState.rjust(15),writeState.rjust(15)) )

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........

      random.shuffle( candidates )
      selChannel, selSourceSE, selTargetSE = candidates[0]
      ancestor = False
      for channelID, treeItem in tree.items():
        if selSourceSE in treeItem["DestSE"]:
          ancestor = channelID
      tree[selChannel.channelID] = { "Ancestor" : ancestor,
                                     "SourceSE" : selSourceSE,
                                     "DestSE" : selTargetSE,
                                     "Strategy" : "DynamicThroughput" }
      timeToSite[selTargetSE] = selTimeToStart 
      sourceSEs.append( selTargetSE )
      targetSEs.remove( selTargetSE )
  
    return S_OK( tree )

  def reset( self ):
    """ reset :chosenStrategy: 

    :param self: self reference
    """
    self.chosenStrategy = 0

  def getSupportedStrategies( self ):
    """ Get supported strategies.

    :param self: self reference
    """    
    return self.supportedStrategies

  def replicationTree( self, sourceSEs, targetSEs, size, strategy=None ):
    """ get replication tree

    :param str lfn: LFN
    :param list sourceSEs: list of sources SE names to use
    :param list targetSEs: liost of target SE names to use
    :param long size: file size
    :param str strategy: strategy name
    """
    ## update SEs rwAccess every rwUpdatePertion timedelta (default 300 s)
    now = datetime.datetime.now()
    if now - self.lastRssUpdate > self.rwUpdatePeriod:
      update = self.updateGraph( rwAccess=True )
      if not update["OK"]:
        self.log.warn("replicationTree: unable to update FTS graph: %s" % update["Message"] )
      else:
        self.lastRssUpdate = now
    ## get strategy
    strategy = strategy if strategy else self.__selectStrategy()
    if strategy not in self.getSupportedStrategies():
      return S_ERROR("replicationTree: unsupported strategy '%s'" % strategy )

    self.log.info( "replicationTree: strategy=%s sourceSEs=%s targetSEs=%s size=%s" %\
                     ( strategy, sourceSEs, targetSEs, size ) )
    ## fire action from dispatcher
    tree = self.strategyDispatcher[strategy]( sourceSEs, targetSEs )
    if not tree["OK"]:
      self.log.error( "replicationTree: %s" % tree["Message"] )
      return tree
    ## update graph edges
    update = self.updateGraph( replicationTree=tree["Value"], size=size )
    if not update["OK"]:
      self.log.error( "replicationTree: unable to update FTS graph: %s" % update["Message"] )
      return update
    return tree
    
  def __selectStrategy( self ):
    """ If more than one active strategy use one after the other.

    :param self: self reference
    """
    chosenStrategy = self.activeStrategies[self.chosenStrategy]
    self.chosenStrategy += 1
    if self.chosenStrategy == self.numberOfStrategies:
      self.chosenStrategy = 0
    return chosenStrategy

  def __getRWAccessForSE( self, seList ):
    """ get RSS R/W for :seList: 

    :param list seList: SE list
    """
    rwDict = dict.fromkeys( seList )
    for se in rwDict:
      rwDict[se] = { "read" : False, "write" : False  }
    rAccess = self.resourceStatus.getStorageElementStatus( seList, statusType = "ReadAccess", default = 'Unknown' )
    if not rAccess["OK"]:
      return rAccess["Message"]
    rAccess = [ k for k, v in rAccess["Value"].items() if "ReadAccess" in v and v["ReadAccess"] in ( "Active", 
                                                                                                     "Degraded" ) ]
    wAccess = self.resourceStatus.getStorageElementStatus( seList, statusType = "WriteAccess", default = 'Unknown' )
    if not wAccess["OK"]:
      return wAccess["Message"]
    wAccess = [ k for k, v in wAccess["Value"].items() if "WriteAccess" in v and v["WriteAccess"] in ( "Active", 
                                                                                                       "Degraded" ) ]
    for se in rwDict:
      rwDict[se]["read"] = se in rAccess
      rwDict[se]["write"] = se in wAccess
    return S_OK( rwDict )

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........

  def __sequenceServerPolicy( self ):
    """
       Every time the this policy is called, return the next server on the list
    """

    fts3server = self.__serverList[self.__nextServerID]
    self.__nextServerID = ( self.__nextServerID + 1 ) % len( self.__serverList )
    return fts3server

  def __randomServerPolicy(self):
    """
      return a random server from the list
    """
    return random.choice( self.__serverList )


  def __chooseFTS3Server( self ):
    """
      Choose the appropriate FTS3 server depending on the policy
    """

    fts3Server = None
    attempt = 0
    # FIXME : need to get real valeu from RSS
    ftsServerStatus = True

    while not fts3Server and attempt < self.maxAttempts:
      if self.__serverPolicy == 'Random':
        fts3Server = self.__randomServerPolicy()
      elif self.__serverPolicy == 'Sequence':
        fts3Server = self.__sequenceServerPolicy()
      elif self.__serverPolicy == 'Failover':
        fts3Server = self.__failoverServerPolicy( attempt = attempt )
      else:
        self.log.error( 'Unknown server policy %s. Using Random instead' % self.__serverPolicy )
        fts3Server = self.__randomServerPolicy()

      if not ftsServerStatus:
        self.log.warn( 'FTS server %s is not in good shape. Choose another one' % fts3Server )
        fts3Server = None
      attempt += 1

        # FIXME : I need to get the FTS server status from RSS
#       ftsStatusFromRss = rss.ftsStatusOrSomethingLikeThat

    if fts3Server:
      return S_OK( fts3Server )

    return S_ERROR ( "Could not find an FTS3 server (max attempt reached)" )

  def findRoute( self, sourceSE, targetSE ):
    """ Find the appropriate route from point A to B
      :param sourceSE : source SE
      :param targetSE : destination SE

      :returns S_OK(FTSRoute)

    """

    fts3server = self.__chooseFTS3Server()

    if not fts3server['OK']:
      return fts3server

    fts3server = fts3server['Value']

    route = FTSRoute( sourceSE, targetSE, fts3server )

    return S_OK( route )

  def isRouteValid( self, route ):
    """
        FIXME: until RSS is ready, I check manually the status
        In FTS3, all routes are valid a priori.
        If a route was not valid for some reason, then FTS would know it
        thanks to the blacklist sent by RSS, and would deal with it itself.
       :param route : FTSRoute

       :returns S_OK or S_ERROR(reason)
    """
    
    rAccess = self.rssClient.getStorageElementStatus( route.sourceSE, "ReadAccess" )
    self.log.debug( "se read %s %s" % ( route.sourceSE, rAccess ) )
    if not rAccess["OK"]:
      self.log.error( rAccess["Message"] )
      return rAccess

    if rAccess["Value"][route.sourceSE]["ReadAccess"] not in ( "Active", "Degraded" ):
      return S_ERROR( "Source SE is not readable" )

    wAccess = self.rssClient.getStorageElementStatus( route.targetSE, "WriteAccess" )
    self.log.debug( "se write %s %s" % ( route.targetSE, wAccess ) )
    if not wAccess["OK"]:
      self.log.error( wAccess["Message"] )
      return wAccess
    if wAccess["Value"][route.targetSE]["WriteAccess"] not in ( "Active", "Degraded" ):
      return S_ERROR( "Target SE is not writable" )

    return S_OK()

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........
    result = self.setOptimizerJobInfo( job, self.am_getModuleParam( 'optimizerName' ), resolvedData )
    if not result['OK']:
      self.log.warn( result['Message'] )
      return result
    return S_OK( resolvedData )


  #############################################################################
  def __getSitesForSE( self, se ):
    """ Returns a list of sites having the given SE as a local one.
        Uses the local cache of the site-se information
    """

    # Empty the cache if too old
    if ( time.time() - self.lastCScheck ) > self.cacheLength:
      self.log.verbose( 'Resetting the SE to site mapping cache' )
      self.seToSiteMapping = {}
      self.lastCScheck = time.time()

    if se not in self.seToSiteMapping:
      sites = getSitesForSE( se )
      if sites['OK']:
        self.seToSiteMapping[se] = list( sites['Value'] )
      return sites
    else:
      return S_OK( self.seToSiteMapping[se] )

  #############################################################################
  def __getSiteCandidates( self, inputData ):
    """This method returns a list of possible site candidates based on the
       job input data requirement.  For each site candidate, the number of files
       on disk and tape is resolved.
    """

    fileSEs = {}
    for lfn, replicas in inputData.items():
      siteList = []
      for se in replicas.keys():
        sites = self.__getSitesForSE( se )
        if sites['OK']:
          siteList += sites['Value']
      fileSEs[lfn] = uniqueElements( siteList )

    siteCandidates = []
    i = 0
    for _fileName, sites in fileSEs.items():
      if not i:
        siteCandidates = sites
      else:
        tempSite = []
        for site in siteCandidates:
          if site in sites:
            tempSite.append( site )
        siteCandidates = tempSite
      i += 1

    if not len( siteCandidates ):
      return S_ERROR( 'No candidate sites available' )

    #In addition, check number of files on tape and disk for each site
    #for optimizations during scheduling
    siteResult = {}
    for site in siteCandidates:
      siteResult[site] = { 'disk': [], 'tape': [] }

    seDict = {}
    for lfn, replicas in inputData.items():
      for se in replicas.keys():
        if se not in seDict:
          sites = self.__getSitesForSE( se )
          if not sites['OK']:
            continue
          try:
            #storageElement = StorageElement( se )
            result = self.resourceStatus.getStorageElementStatus( se, statusType = 'ReadAccess' )
            if not result['OK']:
              continue
            seDict[se] = { 'Sites': sites['Value'], 'SEParams': result['Value'][se] }
            result = getStorageElementOptions( se )
            if not result['OK']:
              continue
            seDict[se]['SEParams'].update(result['Value'])
          except Exception:
            self.log.exception( 'Failed to instantiate StorageElement( %s )' % se )
            continue
        for site in seDict[se]['Sites']:
          if site in siteCandidates:
            if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['DiskSE']:
              if lfn not in siteResult[site]['disk']:
                siteResult[site]['disk'].append( lfn )
                if lfn in siteResult[site]['tape']:
                  siteResult[site]['tape'].remove( lfn )
            if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['TapeSE']:
              if lfn not in siteResult[site]['tape'] and lfn not in siteResult[site]['disk']:
                siteResult[site]['tape'].append( lfn )

    for site in siteResult:
      siteResult[site]['disk'] = len( siteResult[site]['disk'] )
      siteResult[site]['tape'] = len( siteResult[site]['tape'] )
    return S_OK( siteResult )

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........
      tree[selectedChannelID] = { "Ancestor" : waitingChannel,
                                  "SourceSE" : selectedSourceSE,
                                  "DestSE" : selectedDestSE,
                                  "Strategy" : "MinimiseTotalWait" }
      sourceSEs.append( selectedDestSE )
      destSEs.remove( selectedDestSE )
      
    return tree

  def __getTimeToStart( self ):
    """ Generate the dictionary of times to start based on task queue contents and observed throughput.

    :param self: self reference
    """

    if self.schedulingType not in ( "File", "Throughput" ):
      errStr = "__getTimeToStart: CS SchedulingType entry must be either 'File' or 'Throughput'"
      self.log.error( errStr )
      return S_ERROR( errStr )

    channelInfo = {}
    for channelID, bandwidth in self.bandwidths.items():

      channelDict = self.channels[channelID] 
      channelName = channelDict["ChannelName"]

      # initial equal 0.0
      timeToStart = 0.0

      channelStatus = channelDict["Status"]

      ## channel is active?
      if channelStatus == "Active":
        
        channelFileSuccess = bandwidth["SuccessfulFiles"]
        channelFileFailed = bandwidth["FailedFiles"]
        attempted = channelFileSuccess + channelFileFailed
        

        successRate = 100.0
        if attempted != 0:
          successRate = 100.0 * ( channelFileSuccess / float( attempted ) )
    
        ## get distinct failed files counter
        distinctFailedFiles = self.failedFiles.get( channelID, 0 )      
    
        ## success rate too low and more than acceptable distinct files are affected?, make channel unattractive
        if ( successRate < self.acceptableFailureRate ) and ( distinctFailedFiles > self.acceptableFailedFiles ):
          timeToStart = float( "inf" ) 
        else:

          ## scheduling type == Throughput
          transferSpeed = bandwidth["Throughput"] 
          waitingTransfers = channelDict["Size"]

          ## scheduling type == File, overwrite transferSpeed and waitingTransfer
          if self.schedulingType == "File":
            transferSpeed = bandwidth["Fileput"] 
            waitingTransfers = channelDict["Files"]

          if transferSpeed > 0:
            timeToStart = waitingTransfers / float( transferSpeed )
            
      else:
        ## channel not active, make it unattractive
        timeToStart = float( "inf" ) 

      channelInfo.setdefault( channelName, { "ChannelID" : channelID, 
                                             "TimeToStart" : timeToStart } )

    return S_OK( channelInfo )

  def __getActiveSEs( self, seList, access = "Read" ):
    """Get active storage elements.

    :param self: self reference
    :param list seList: stogare element list
    :param str access: storage element accesss, could be 'Read' (default) or 'Write' 
    """
    res = self.resourceStatus.getStorageElementStatus( seList, statusType = access, default = 'Unknown' )
    if not res["OK"]:
      return []
    return [ k for k, v in res["Value"].items() if access in v and v[access] in ( "Active", "Bad" ) ]
   
  def __getChannelSitesForSE( self, storageElement ):
    """Get sites for given storage element.
    
    :param self: self reference
    :param str storageElement: storage element name
    """
    res = getSitesForSE( storageElement )
    if not res["OK"]:
      return []
    sites = []
    for site in res["Value"]:
      siteName = site.split( "." )
      if len( siteName ) > 1:
        if not siteName[1] in sites:
          sites.append( siteName[1] )
    return sites

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatus.ResourceStatus import getStorageElementStatus [as 别名]

#.........这里部分代码省略.........
      gLogger.error( errStr, configPath )
      return S_ERROR( errStr )
    if 'Alias' in res['Value']:
      configPath = '%s/%s/Alias' % ( self.rootConfigPath, storageName )
      aliasName = gConfig.getValue( configPath )
      result = self._getConfigStorageName( aliasName )
      if not result['OK']:
        errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
        gLogger.error( errStr, configPath )
        return S_ERROR( errStr )
      resolvedName = result['Value']
    else:
      resolvedName = storageName
    return S_OK( resolvedName )

  def _getConfigStorageOptions( self, storageName ):
    """ Get the options associated to the StorageElement as defined in the CS
    """
    storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName )
    res = gConfig.getOptions( storageConfigPath )
    if not res['OK']:
      errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )
    options = res['Value']
    optionsDict = {}
    for option in options:

      if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']:
        continue
      optionConfigPath = '%s/%s' % ( storageConfigPath, option )
      optionsDict[option] = gConfig.getValue( optionConfigPath, '' )

    res = self.resourceStatus.getStorageElementStatus( storageName )
    if not res[ 'OK' ]:
      errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )

    # For safety, we did not add the ${statusType}Access keys
    # this requires modifications in the StorageElement class

    # We add the dictionary with the statusTypes and values
    # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
    optionsDict.update( res[ 'Value' ][ storageName ] )

    return S_OK( optionsDict )

  def _getConfigStorageProtocols( self, storageName ):
    """ Protocol specific information is present as sections in the Storage configuration
    """
    storageConfigPath = '%s/%s' % ( self.rootConfigPath, storageName )
    res = gConfig.getSections( storageConfigPath )
    if not res['OK']:
      errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections"
      gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
      return S_ERROR( errStr )
    protocolSections = res['Value']
    sortedProtocols = sortList( protocolSections )
    protocolDetails = []
    for protocol in sortedProtocols:
      res = self._getConfigStorageProtocolDetails( storageName, protocol )
      if not res['OK']:
        return res
      protocolDetails.append( res['Value'] )
    self.protocols = self.localProtocols + self.remoteProtocols