Python ResourceStatus.ResourceStatus类代码示例

  def __init__( self, useProxy=False ):

    self.rootConfigPath = '/Resources/StorageElements'
    self.valid = True
    self.proxy = False
    self.proxy = useProxy
    self.resourceStatus = ResourceStatus()

    def __init__(self, useProxy=False, vo=None):

        self.valid = True
        self.proxy = False
        self.proxy = useProxy
        self.resourceStatus = ResourceStatus()
        self.resourcesHelper = Resources(vo=vo)

  def __init__( self, configSection, channels=None, bandwidths=None, failedFiles=None ):
    """c'tor

    :param self: self reference
    :param str configSection: path on CS to ReplicationScheduler agent
    :param bandwithds: observed throughput on active channels
    :param channels: active channels
    :param int failedFiles: max number of distinct failed files to allow scheduling
    """
    ## save config section
    self.configSection = configSection + "/" + self.__class__.__name__
    ## 

    ## sublogger
    self.log = gLogger.getSubLogger( "StrategyHandler", child=True )
    self.log.setLevel( gConfig.getValue( self.configSection + "/LogLevel", "DEBUG"  ) )
  
    self.supportedStrategies = [ 'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait' ]
    self.log.info( "Supported strategies = %s" % ", ".join( self.supportedStrategies ) )
  
    self.sigma = gConfig.getValue( self.configSection + '/HopSigma', 0.0 )
    self.log.info( "HopSigma = %s" % self.sigma )
    self.schedulingType = gConfig.getValue( self.configSection + '/SchedulingType', 'File' )
    self.log.info( "SchedulingType = %s" % self.schedulingType )
    self.activeStrategies = gConfig.getValue( self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'] )
    self.log.info( "ActiveStrategies = %s" % ", ".join( self.activeStrategies ) )
    self.numberOfStrategies = len( self.activeStrategies )
    self.log.info( "Number of active strategies = %s" % self.numberOfStrategies )
    self.acceptableFailureRate = gConfig.getValue( self.configSection + '/AcceptableFailureRate', 75 )
    self.log.info( "AcceptableFailureRate = %s" % self.acceptableFailureRate )
    self.acceptableFailedFiles = gConfig.getValue( self.configSection + "/AcceptableFailedFiles", 5 )
    self.log.info( "AcceptableFailedFiles = %s" % self.acceptableFailedFiles )
    self.rwUpdatePeriod = gConfig.getValue( self.configSection + "/RssRWUpdatePeriod", 300 )
    self.log.info( "RSSUpdatePeriod = %s s" % self.rwUpdatePeriod )
    self.rwUpdatePeriod = datetime.timedelta( seconds=self.rwUpdatePeriod )
    ## bandwithds
    self.bandwidths = bandwidths if bandwidths else {}
    ## channels
    self.channels = channels if channels else {}
    ## distinct failed files per channel 
    self.failedFiles = failedFiles if failedFiles else {}
    ## chosen strategy
    self.chosenStrategy = 0
    ## fts graph
    self.ftsGraph = None
    ## timestamp for last update
    self.lastRssUpdate = datetime.datetime.now()    
    # dispatcher
    self.strategyDispatcher = { "MinimiseTotalWait" : self.minimiseTotalWait, 
                                "DynamicThroughput" : self.dynamicThroughput,
                                "Simple" : self.simple, 
                                "Swarm" : self.swarm }
    ## own RSS client
    self.resourceStatus = ResourceStatus()
    ## create fts graph
    ftsGraph = self.setup( self.channels, self.bandwidths, self.failedFiles )    
    if not ftsGraph["OK"]:
      raise SHGraphCreationError( ftsGraph["Message"] )
    self.log.info("%s has been constructed" % self.__class__.__name__ )

  def __init__( self ):

    self.rootConfigPath = '/Resources/StorageElements'
    self.valid = True
    self.proxy = False
    res = gConfig.getOption( "%s/UseProxy" % self.rootConfigPath )
    if res['OK'] and ( res['Value'] == 'True' ):
      self.proxy = True
    self.resourceStatus = ResourceStatus()

  def __init__( self, configSection, bandwidths=None, channels=None, failedFiles=None ):
    """c'tor

    :param self: self reference
    :param str configSection: path on CS to ReplicationScheduler agent
    :param bandwithds: observed throughput on active channels
    :param channels: active channels
    :param int failedFiles: max number of distinct failed files to allow scheduling
    """
    ## save config section
    self.configSection = configSection + "/" + self.__class__.__name__
    ## sublogger
    self.log = gLogger.getSubLogger( "StrategyHandler", child=True )
    self.log.setLevel( gConfig.getValue( self.configSection + "/LogLevel", "DEBUG"  ) )
  
    self.supportedStrategies = [ 'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait' ]
    self.log.debug( "Supported strategies = %s" % ", ".join( self.supportedStrategies ) )
  
    self.sigma = gConfig.getValue( self.configSection + '/HopSigma', 0.0 )
    self.log.debug( "HopSigma = %s" % self.sigma )
    self.schedulingType = gConfig.getValue( self.configSection + '/SchedulingType', 'File' )
    self.log.debug( "SchedulingType = %s" % self.schedulingType )
    self.activeStrategies = gConfig.getValue( self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'] )
    self.log.debug( "ActiveStrategies = %s" % ", ".join( self.activeStrategies ) )
    self.numberOfStrategies = len( self.activeStrategies )
    self.log.debug( "Number of active strategies = %s" % self.numberOfStrategies )
    self.acceptableFailureRate = gConfig.getValue( self.configSection + '/AcceptableFailureRate', 75 )
    self.log.debug( "AcceptableFailureRate = %s" % self.acceptableFailureRate )
    self.acceptableFailedFiles = gConfig.getValue( self.configSection + "/AcceptableFailedFiles", 5 )
    self.log.debug( "AcceptableFailedFiles = %s" % self.acceptableFailedFiles )

    self.bandwidths = bandwidths if bandwidths else {}
    self.channels = channels if channels else {}
    self.failedFiles = failedFiles if failedFiles else {}
    self.chosenStrategy = 0

    # dispatcher
    self.strategyDispatcher = { re.compile("MinimiseTotalWait") : self.__minimiseTotalWait, 
                                re.compile("DynamicThroughput") : self.__dynamicThroughput,
                                re.compile("Simple") : self.__simple, 
                                re.compile("Swarm") : self.__swarm }

    self.resourceStatus = ResourceStatus()

    self.log.debug( "strategyDispatcher entries:" )
    for key, value in self.strategyDispatcher.items():
      self.log.debug( "%s : %s" % ( key.pattern, value.__name__ ) )

    self.log.debug("%s has been constructed" % self.__class__.__name__ )

  def __init__( self, csPath = None, ftsHistoryViews = None ):
    """
        Call the init of the parent, and initialize the list of FTS3 servers
    """

    self.log = gLogger.getSubLogger( "FTS3Placement" )
    super( FTS3Placement, self ).__init__( csPath = csPath, ftsHistoryViews = ftsHistoryViews )
    srvList = getFTS3Servers()
    if not srvList['OK']:
      self.log.error( srvList['Message'] )

    self.__serverList = srvList.get( 'Value', [] )
    self.maxAttempts = len( self.__serverList )

    self.rssClient = ResourceStatus()

  def initializeOptimizer( self ):
    """Initialize specific parameters for JobSanityAgent.
    """
    self.failedMinorStatus = self.am_getOption( '/FailedJobStatus', 'Input Data Not Available' )
    #this will ignore failover SE files
    self.checkFileMetadata = self.am_getOption( 'CheckFileMetadata', True )

    self.dataManager = DataManager()
    self.resourceStatus = ResourceStatus()
    self.fc = FileCatalog()

    self.seToSiteMapping = {}
    self.lastCScheck = 0
    self.cacheLength = 600

    return S_OK()

 def __init__( self, useProxy = False, vo = None ):
   self.rootConfigPath = '/Resources/StorageElements'
   self.proxy = False
   self.proxy = useProxy
   self.resourceStatus = ResourceStatus()
   self.vo = vo
   if self.vo is None:
     result = getVOfromProxyGroup()
     if result['OK']:
       self.vo = result['Value']
     else:
       RuntimeError( "Can not get the current VO context" )
   self.remotePlugins = []
   self.localPlugins = []
   self.name = ''
   self.options = {}
   self.protocolDetails = []
   self.storages = []

  def __init__(self, serverDict, serverPolicy="Random"):
    """
        Call the init of the parent, and initialize the list of FTS3 servers
    """

    self.log = gLogger.getSubLogger("FTS3ServerPolicy")

    self._serverDict = serverDict
    self._serverList = serverDict.keys()
    self._maxAttempts = len(self._serverList)
    self._nextServerID = 0
    self._resourceStatus = ResourceStatus()

    methName = "_%sServerPolicy" % serverPolicy.lower()
    if not hasattr(self, methName):
      self.log.error('Unknown server policy %s. Using Random instead' % serverPolicy)
      methName = "_randomServerPolicy"

    self._policyMethod = getattr(self, methName)

class StorageFactory:

  def __init__( self, useProxy = False, vo = None ):

    self.rootConfigPath = '/Resources/StorageElements'
    self.valid = True
    self.proxy = False
    self.proxy = useProxy
    self.resourceStatus = ResourceStatus()
    self.vo = vo

  ###########################################################################################
  #
  # Below are public methods for obtaining storage objects
  #

  def getStorageName( self, initialName ):
    return self._getConfigStorageName( initialName )

  def getStorage( self, parameterDict ):
    """ This instantiates a single storage for the details provided and doesn't check the CS.
    """
    # The storage name must be supplied.
    if parameterDict.has_key( 'StorageName' ):
      storageName = parameterDict['StorageName']
    else:
      errStr = "StorageFactory.getStorage: StorageName must be supplied"
      gLogger.error( errStr )
      return S_ERROR( errStr )

    # ProtocolName must be supplied otherwise nothing with work.
    if parameterDict.has_key( 'ProtocolName' ):
      protocolName = parameterDict['ProtocolName']
    else:
      errStr = "StorageFactory.getStorage: ProtocolName must be supplied"
      gLogger.error( errStr )
      return S_ERROR( errStr )

    # The other options need not always be specified
    if parameterDict.has_key( 'Protocol' ):
      protocol = parameterDict['Protocol']
    else:
      protocol = ''

    if parameterDict.has_key( 'Port' ):
      port = parameterDict['Port']
    else:
      port = ''

    if parameterDict.has_key( 'Host' ):
      host = parameterDict['Host']
    else:
      host = ''

    if parameterDict.has_key( 'Path' ):
      path = parameterDict['Path']
    else:
      path = ''

    if parameterDict.has_key( 'SpaceToken' ):
      spaceToken = parameterDict['SpaceToken']
    else:
      spaceToken = ''

    if parameterDict.has_key( 'WSUrl' ):
      wsPath = parameterDict['WSUrl']
    else:
      wsPath = ''

    return self.__generateStorageObject( storageName, protocolName, protocol, path, host, port, spaceToken, wsPath, parameterDict )


  def getStorages( self, storageName, protocolList = [] ):
    """ Get an instance of a Storage based on the DIRAC SE name based on the CS entries CS

        'storageName' is the DIRAC SE name i.e. 'CERN-RAW'
        'protocolList' is an optional list of protocols if a sub-set is desired i.e ['SRM2','SRM1']
    """
    self.remoteProtocols = []
    self.localProtocols = []
    self.name = ''
    self.options = {}
    self.protocolDetails = []
    self.storages = []

    # Get the name of the storage provided
    res = self._getConfigStorageName( storageName )
    if not res['OK']:
      self.valid = False
      return res
    storageName = res['Value']
    self.name = storageName

    # Get the options defined in the CS for this storage
    res = self._getConfigStorageOptions( storageName )
    if not res['OK']:
      self.valid = False
      return res
    self.options = res['Value']

#.........这里部分代码省略.........

class StrategyHandler( object ):
  """
  .. class:: StrategyHandler

  StrategyHandler is a helper class for determining optimal replication tree for given
  source files, their replicas and target storage elements.
  """

  def __init__( self, configSection, channels=None, bandwidths=None, failedFiles=None ):
    """c'tor

    :param self: self reference
    :param str configSection: path on CS to ReplicationScheduler agent
    :param bandwithds: observed throughput on active channels
    :param channels: active channels
    :param int failedFiles: max number of distinct failed files to allow scheduling
    """
    ## save config section
    self.configSection = configSection + "/" + self.__class__.__name__
    ## 

    ## sublogger
    self.log = gLogger.getSubLogger( "StrategyHandler", child=True )
    self.log.setLevel( gConfig.getValue( self.configSection + "/LogLevel", "DEBUG"  ) )
  
    self.supportedStrategies = [ 'Simple', 'DynamicThroughput', 'Swarm', 'MinimiseTotalWait' ]
    self.log.info( "Supported strategies = %s" % ", ".join( self.supportedStrategies ) )
  
    self.sigma = gConfig.getValue( self.configSection + '/HopSigma', 0.0 )
    self.log.info( "HopSigma = %s" % self.sigma )
    self.schedulingType = gConfig.getValue( self.configSection + '/SchedulingType', 'File' )
    self.log.info( "SchedulingType = %s" % self.schedulingType )
    self.activeStrategies = gConfig.getValue( self.configSection + '/ActiveStrategies', ['MinimiseTotalWait'] )
    self.log.info( "ActiveStrategies = %s" % ", ".join( self.activeStrategies ) )
    self.numberOfStrategies = len( self.activeStrategies )
    self.log.info( "Number of active strategies = %s" % self.numberOfStrategies )
    self.acceptableFailureRate = gConfig.getValue( self.configSection + '/AcceptableFailureRate', 75 )
    self.log.info( "AcceptableFailureRate = %s" % self.acceptableFailureRate )
    self.acceptableFailedFiles = gConfig.getValue( self.configSection + "/AcceptableFailedFiles", 5 )
    self.log.info( "AcceptableFailedFiles = %s" % self.acceptableFailedFiles )
    self.rwUpdatePeriod = gConfig.getValue( self.configSection + "/RssRWUpdatePeriod", 300 )
    self.log.info( "RSSUpdatePeriod = %s s" % self.rwUpdatePeriod )
    self.rwUpdatePeriod = datetime.timedelta( seconds=self.rwUpdatePeriod )
    ## bandwithds
    self.bandwidths = bandwidths if bandwidths else {}
    ## channels
    self.channels = channels if channels else {}
    ## distinct failed files per channel 
    self.failedFiles = failedFiles if failedFiles else {}
    ## chosen strategy
    self.chosenStrategy = 0
    ## fts graph
    self.ftsGraph = None
    ## timestamp for last update
    self.lastRssUpdate = datetime.datetime.now()    
    # dispatcher
    self.strategyDispatcher = { "MinimiseTotalWait" : self.minimiseTotalWait, 
                                "DynamicThroughput" : self.dynamicThroughput,
                                "Simple" : self.simple, 
                                "Swarm" : self.swarm }
    ## own RSS client
    self.resourceStatus = ResourceStatus()
    ## create fts graph
    ftsGraph = self.setup( self.channels, self.bandwidths, self.failedFiles )    
    if not ftsGraph["OK"]:
      raise SHGraphCreationError( ftsGraph["Message"] )
    self.log.info("%s has been constructed" % self.__class__.__name__ )

  def setup( self, channels, bandwithds, failedFiles ):
    """ prepare fts graph 

    :param dict channels: { channelID : { "Files" : long , Size = long, "ChannelName" : str, 
                                          "Source" : str, "Destination" : str , "ChannelName" : str, "Status" : str  } }
    :param dict bandwidths: { channelID { "Throughput" : float, "Fileput" : float, "SucessfulFiles" : long, "FailedFiles" : long  } }
    :param dict failedFiles: { channelID : int }

    channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }  
    """
    graph = FTSGraph( "sites" )
   
    result = getStorageElementSiteMapping()
    if not result['OK']:
      return result
    sitesDict = result['Value']

    ## create nodes 
    for site, ses in sitesDict.items():
      rwDict = self.__getRWAccessForSE( ses )
      if not rwDict["OK"]:
        return rwDict
      siteName = site
      if '.' in site:
        siteName = site.split('.')[1]  
      graph.addNode( LCGSite( siteName, { "SEs" : rwDict["Value"] } ) )
    ## channels { channelID : { "Files" : long , Size = long, "ChannelName" : str, 
    ##                          "Source" : str, "Destination" : str , 
    ##                          "ChannelName" : str, "Status" : str  } }
    ## bandwidths { channelID { "Throughput" : float, "Fileput" : float, 
    ##                           "SucessfulFiles" : long, "FailedFiles" : long  } }
    ## channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
#.........这里部分代码省略.........

class FTS3Placement( FTSAbstractPlacement ):

  """
  This class manages all the FTS strategies, routes and what not
  """


  __serverPolicy = "Random"
  __nextServerID = 0
  __serverList = None
  __maxAttempts = 0


  def __init__( self, csPath = None, ftsHistoryViews = None ):
    """
        Call the init of the parent, and initialize the list of FTS3 servers
    """

    self.log = gLogger.getSubLogger( "FTS3Placement" )
    super( FTS3Placement, self ).__init__( csPath = csPath, ftsHistoryViews = ftsHistoryViews )
    srvList = getFTS3Servers()
    if not srvList['OK']:
      self.log.error( srvList['Message'] )

    self.__serverList = srvList.get( 'Value', [] )
    self.maxAttempts = len( self.__serverList )

    self.rssClient = ResourceStatus()



  def getReplicationTree( self, sourceSEs, targetSEs, size, strategy = None ):
    """ For multiple source to multiple destination, find the optimal replication
        strategy.

       :param sourceSEs : list of source SE
       :param targetSEs : list of destination SE
       :param size : size of the File
       :param strategy : which strategy to use

       :returns S_OK(dict) < route name :  { dict with key Ancestor, SourceSE, TargetSEtargetSE, Strategy } >

       For the time being, we are waiting for FTS3 to provide advisory mechanisms. So we just use
       simple techniques
    """

    # We will use a single random source
    sourceSE = random.choice( sourceSEs )

    tree = {}
    for targetSE in targetSEs:
      tree["%s#%s" % ( sourceSE, targetSE )] = { "Ancestor" : False, "SourceSE" : sourceSE,
                           "TargetSE" : targetSE, "Strategy" : "FTS3Simple" }

    return S_OK( tree )



  def refresh( self, ftsHistoryViews ):
    """
    Refresh, whatever that means... recalculate all what you need,
    fetches the latest conf and what not.
    """
    return super( FTS3Placement, self ).refresh( ftsHistoryViews = ftsHistoryViews )



  def __failoverServerPolicy(self, attempt = 0):
    """
       Returns always the server at a given position (normally the first one)

       :param attempt: position of the server in the list
    """
    if attempt >= len( self.__serverList ):
      raise Exception( "FTS3Placement.__failoverServerPolicy: attempt to reach non existing server index" )

    return self.__serverList[attempt]

  def __sequenceServerPolicy( self ):
    """
       Every time the this policy is called, return the next server on the list
    """

    fts3server = self.__serverList[self.__nextServerID]
    self.__nextServerID = ( self.__nextServerID + 1 ) % len( self.__serverList )
    return fts3server

  def __randomServerPolicy(self):
    """
      return a random server from the list
    """
    return random.choice( self.__serverList )


  def __chooseFTS3Server( self ):
    """
      Choose the appropriate FTS3 server depending on the policy
    """

    fts3Server = None
#.........这里部分代码省略.........

if __name__ == "__main__":
  
  result = getVOfromProxyGroup()
  if not result['OK']:
    gLogger.notice( 'Error:', result['Message'] )
    DIRAC.exit( 1 )
  vo = result['Value']  
  resources = Resources( vo = vo )
  result = resources.getEligibleStorageElements()
  if not result['OK']:
    gLogger.notice( 'Error:', result['Message'] )
    DIRAC.exit( 2 )
  seList = sortList( result[ 'Value' ] )

  resourceStatus = ResourceStatus()
 
  result = resourceStatus.getStorageStatus( seList )
  if not result['OK']:
    gLogger.notice( 'Error:', result['Message'] )
    DIRAC.exit( 3 )

  for k,v in result[ 'Value' ].items():
    
    readState, writeState = 'Active', 'Active'
    
    if v.has_key( 'ReadAccess' ):
      readState = v[ 'ReadAccess' ]  
    
    if v.has_key( 'WriteAccess' ):
      writeState = v[ 'WriteAccess']

class FTS3ServerPolicy(object):
  """
  This class manages the policy for choosing a server
  """

  def __init__(self, serverDict, serverPolicy="Random"):
    """
        Call the init of the parent, and initialize the list of FTS3 servers
    """

    self.log = gLogger.getSubLogger("FTS3ServerPolicy")

    self._serverDict = serverDict
    self._serverList = serverDict.keys()
    self._maxAttempts = len(self._serverList)
    self._nextServerID = 0
    self._resourceStatus = ResourceStatus()

    methName = "_%sServerPolicy" % serverPolicy.lower()
    if not hasattr(self, methName):
      self.log.error('Unknown server policy %s. Using Random instead' % serverPolicy)
      methName = "_randomServerPolicy"

    self._policyMethod = getattr(self, methName)

  def _failoverServerPolicy(self, _attempt):
    """
       Returns always the server at a given position (normally the first one)

       :param attempt: position of the server in the list
    """
    if _attempt >= len(self._serverList):
      raise Exception(
          "FTS3ServerPolicy.__failoverServerPolicy: attempt to reach non existing server index")
    return self._serverList[_attempt]

  def _sequenceServerPolicy(self, _attempt):
    """
       Every time the this policy is called, return the next server on the list
    """

    fts3server = self._serverList[self._nextServerID]
    self._nextServerID = (self._nextServerID + 1) % len(self._serverList)
    return fts3server

  def _randomServerPolicy(self, _attempt):
    """
      return a server from shuffledServerList
    """

    if getattr(threadLocal, 'shuffledServerList', None) is None:
      threadLocal.shuffledServerList = self._serverList[:]
      random.shuffle(threadLocal.shuffledServerList)

    fts3Server = threadLocal.shuffledServerList[_attempt]

    if _attempt == self._maxAttempts - 1:
      random.shuffle(threadLocal.shuffledServerList)

    return fts3Server

  def _getFTSServerStatus(self, ftsServer):
    """ Fetch the status of the FTS server from RSS """

    res = self._resourceStatus.getElementStatus(ftsServer, 'FTS')
    if not res['OK']:
      return res

    result = res['Value']
    if ftsServer not in result:
      return S_ERROR("No FTS Server %s known to RSS" % ftsServer)

    if result[ftsServer]['all'] == 'Active':
      return S_OK(True)

    return S_OK(False)

  def chooseFTS3Server(self):
    """
      Choose the appropriate FTS3 server depending on the policy
    """

    fts3Server = None
    attempt = 0

    while not fts3Server and attempt < self._maxAttempts:

      fts3Server = self._policyMethod(attempt)
      res = self._getFTSServerStatus(fts3Server)

      if not res['OK']:
        self.log.warn("Error getting the RSS status for %s: %s" % (fts3Server, res))
        fts3Server = None
        attempt += 1
        continue

      ftsServerStatus = res['Value']

      if not ftsServerStatus:
        self.log.warn('FTS server %s is not in good shape. Choose another one' % fts3Server)
#.........这里部分代码省略.........

if site:
  res = gConfig.getOptionsDict( '/Resources/Sites/LCG/%s' % site )
  if not res[ 'OK' ]:
    gLogger.error( 'The provided site (%s) is not known.' % site )
    DIRAC.exit( -1 )
  ses.extend( res[ 'Value' ][ 'SE' ].replace( ' ', '' ).split( ',' ) )
if not ses:
  gLogger.error( 'There were no SEs provided' )
  DIRAC.exit()

readAllowed = []
writeAllowed = []
checkAllowed = []
removeAllowed = []

resourceStatus = ResourceStatus()

res = resourceStatus.getStorageElementStatus( ses )
if not res[ 'OK' ]:
  gLogger.error( 'Storage Element %s does not exist' % ses )
  DIRAC.exit( -1 )

reason = 'Forced with dirac-admin-allow-se by %s' % userName

for se, seOptions in res[ 'Value' ].items():

  resW = resC = resR = { 'OK' : False }


  # InActive is used on the CS model, Banned is the equivalent in RSS
  if read and seOptions.has_key( 'ReadAccess' ):