Python DictCache.getKeys方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.DictCache import DictCache [as 别名]
# 或者: from DIRAC.Core.Utilities.DictCache.DictCache import getKeys [as 别名]
class RSSCache( object ):
  '''
  Cache with purgeThread integrated
  '''

  def __init__( self, lifeTime, updateFunc = None, cacheHistoryLifeTime = None ):
    '''
    Constructor
    '''

    self.__lifeTime             = lifeTime
    # lifetime of the history on hours
    self.__cacheHistoryLifeTime = ( 1 and cacheHistoryLifeTime ) or 24
    self.__updateFunc           = updateFunc

    # RSSCache
    self.__rssCache       = DictCache()
    self.__rssCacheStatus = [] # ( updateTime, message )
    self.__rssCacheLock   = threading.Lock()

    # Create purgeThread
    self.__refreshStop    = False
    self.__refreshThread  = threading.Thread( target = self.__refreshCacheThreadRun )
    self.__refreshThread.setDaemon( True )

  def startRefreshThread( self ):
    '''
    Run refresh thread.
    '''
    self.__refreshThread.start()

  def stopRefreshThread( self ):
    '''
    Stop refresh thread.
    '''
    self.__refreshStop = True

  def isCacheAlive( self ):
    '''
    Returns status of the cache refreshing thread
    '''
    return S_OK( self.__refreshThread.isAlive() )

  def setLifeTime( self, lifeTime ):
    '''
    Set cache life time
    '''
    self.__lifeTime = lifeTime

  def setCacheHistoryLifeTime( self, cacheHistoryLifeTime ):
    '''
    Set cache life time
    '''
    self.__cacheHistoryLifeTime = cacheHistoryLifeTime

  def getCacheKeys( self ):
    '''
    List all the keys stored in the cache.
    '''
    self.__rssCacheLock.acquire()
    keys = self.__rssCache.getKeys()
    self.__rssCacheLock.release()

    return S_OK( keys )

  def acquireLock( self ):
    '''
    Acquires RSSCache lock
    '''
    self.__rssCacheLock.acquire()

  def releaseLock( self ):
    '''
    Releases RSSCache lock
    '''
    self.__rssCacheLock.release()

  def getCacheStatus( self ):
    '''
    Return the latest cache status
    '''
    self.__rssCacheLock.acquire()
    if self.__rssCacheStatus:
      res = dict( [ self.__rssCacheStatus[ 0 ] ] )
    else:
      res = {}
    self.__rssCacheLock.release()
    return S_OK( res )

  def getCacheHistory( self ):
    '''
    Return the cache updates history
    '''
    self.__rssCacheLock.acquire()
    res = dict( self.__rssCacheStatus )
    self.__rssCacheLock.release()
    return S_OK( res )

  def get( self, resourceKey ):
    '''
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.DictCache import DictCache [as 别名]
# 或者: from DIRAC.Core.Utilities.DictCache.DictCache import getKeys [as 别名]
class DIRACPilotDirector(PilotDirector):
  """
    DIRAC PilotDirector class
  """
  def __init__( self, submitPool ):
    """
     Define some defaults and call parent __init__
    """
    self.gridMiddleware    = 'DIRAC'

    PilotDirector.__init__( self, submitPool )

    self.computingElementList = COMPUTING_ELEMENTS
    self.computingElementDict = {}
    self.addComputingElement( self.computingElementList )

    self.siteName          = gConfig.getValue('/LocalSite/Site','')
    if not self.siteName:
      self.log.error( 'Can not run a Director if Site Name is not defined' )
      sys.exit()

    self.__failingCECache  = DictCache()
    self.__ticketsCECache  = DictCache()

  def configure(self, csSection, submitPool ):
    """
     Here goes common configuration for DIRAC PilotDirector
    """

    PilotDirector.configure( self, csSection, submitPool )
    self.reloadConfiguration( csSection, submitPool )

    self.__failingCECache.purgeExpired()
    self.__ticketsCECache.purgeExpired()

    for ce in self.__failingCECache.getKeys():
      if ce in self.computingElementDict.keys():
        try:
          del self.computingElementDict[ce]
        except:
          pass
    if self.computingElementDict:
      self.log.info( ' ComputingElements:', ', '.join(self.computingElementDict.keys()) )
    else:
      return

    # FIXME: this is to start testing
    _ceName, computingElementDict = self.computingElementDict.items()[0]

    self.computingElement = computingElementDict['CE']

    self.log.debug( self.computingElement.getCEStatus() )

    self.log.info( ' SiteName:', self.siteName )


  def configureFromSection( self, mySection ):
    """
      reload from CS
    """
    PilotDirector.configureFromSection( self, mySection )

    self.computingElementList = gConfig.getValue( mySection+'/ComputingElements'      , self.computingElementList )
    self.addComputingElement( self.computingElementList )

    self.siteName             = gConfig.getValue( mySection+'/SiteName'               , self.siteName )


  def addComputingElement(self, ceList):
    """
      Check if a CE object for the current CE is available,
      instantiate one if necessary
    """
    for CE in ceList:
      if CE not in self.computingElementDict:
        ceFactory = ComputingElementFactory( )
        ceInstance = ceFactory.getCE( ceName = CE )
        if not ceInstance['OK']:
          self.log.error('Can not create CE object:', ceInstance['Message'])
          return
        self.computingElementDict[CE] = ceInstance['Value'].ceConfigDict
        # add the 'CE' instance at the end to avoid being overwritten
        self.computingElementDict[CE]['CE'] = ceInstance['Value']


  def _submitPilots( self, workDir, taskQueueDict, pilotOptions, pilotsToSubmit,
                     ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob ):
    """
      This method does the actual pilot submission to the DIRAC CE
      The logic is as follows:
      - If there are no available CE it return error
      - If there is no queue available in the CE's, it returns error
      - It creates a temp directory
      - It prepare a PilotScript
    """

    taskQueueID = taskQueueDict['TaskQueueID']
#     ownerDN = taskQueueDict['OwnerDN']

    submittedPilots = 0
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.DictCache import DictCache [as 别名]
# 或者: from DIRAC.Core.Utilities.DictCache.DictCache import getKeys [as 别名]
class Cache( object ):
  """
    Cache basic class.
    
    WARNING: None of its methods is thread safe. Acquire / Release lock when
    using them !
  """
  
  def __init__( self, lifeTime, updateFunc ):
    """
    Constructor
    
    :Parameters:
      **lifeTime** - `int`
        Lifetime of the elements in the cache ( seconds ! )
      **updateFunc** - `function`
        This function MUST return a S_OK | S_ERROR object. In the case of the first,
        its value must be a dictionary.
    
    """
    
    # We set a 20% of the lifetime randomly, so that if we have thousands of jobs
    # starting at the same time, all the caches will not end at the same time.
    randomLifeTimeBias  = 0.2 * random.random()
    
    self.log            = gLogger.getSubLogger( self.__class__.__name__ )
    
    self.__lifeTime     = int( lifeTime * ( 1 + randomLifeTimeBias ) )
    self.__updateFunc   = updateFunc
    # The records returned from the cache must be valid at least 10 seconds.
    self.__validSeconds = 10
    
    # Cache
    self.__cache       = DictCache()
    self.__cacheLock   = LockRing()
    self.__cacheLock.getLock( self.__class__.__name__ )
  
  #.............................................................................
  # internal cache object getter
  
  def cacheKeys( self ):
    """
    Cache keys getter
      
    :returns: list with valid keys on the cache
    """
    
    return self.__cache.getKeys( validSeconds = self.__validSeconds ) 

  #.............................................................................
  # acquire / release Locks

  def acquireLock( self ):
    """
    Acquires Cache lock
    """
    
    self.__cacheLock.acquire( self.__class__.__name__ )

  def releaseLock( self ):
    """
    Releases Cache lock
    """
    
    self.__cacheLock.release( self.__class__.__name__)
  
  #.............................................................................
  # Cache getters

  def get( self, cacheKeys ):
    """
    Gets values for cacheKeys given, if all are found ( present on the cache and
    valid ), returns S_OK with the results. If any is not neither present not
    valid, returns S_ERROR. 
    
    :Parameters:
      **cacheKeys** - `list`
        list of keys to be extracted from the cache
        
    :return: S_OK | S_ERROR
    """

    result = {}

    for cacheKey in cacheKeys:

      cacheRow = self.__cache.get( cacheKey, validSeconds = self.__validSeconds )
      if not cacheRow:
        self.log.error( str( cacheKey ) )
        return S_ERROR( 'Cannot get %s' % str( cacheKey ) )
      result.update( { cacheKey : cacheRow } )
      
    return S_OK( result )

  #.............................................................................
  # Cache refreshers

  def refreshCache( self ):
    """     
    Purges the cache and gets fresh data from the update function.
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.DictCache import DictCache [as 别名]
# 或者: from DIRAC.Core.Utilities.DictCache.DictCache import getKeys [as 别名]
class CredentialsClient:

  CONSUMER_GRACE_TIME = 3600
  REQUEST_GRACE_TIME = 900

  def __init__( self, RPCFunctor = None ):
    if not RPCFunctor:
      self.__RPCFunctor = RPCClient
    else:
      self.__RPCFunctor = RPCFunctor
    self.__tokens = DictCache()
    self.__requests = DictCache()
    self.__consumers = DictCache( deleteFunction = self.__cleanConsumerCache )

  def __getRPC( self ):
    return self.__RPCFunctor( "WebAPI/Credentials" )

  def __cleanReturn( self, result ):
    if 'rpcStub' in result:
      result.pop( 'rpcStub' )
    return result

  ##
  # Consumer
  ##

  def generateConsumerPair( self, name, callback, icon, consumerKey = "" ):
    result = self.__getRPC().generateConsumerPair( name, callback, icon, consumerKey )
    if not result[ 'OK' ]:
      return self.__cleanReturn( result )
    self.__consumers.add( consumerKey, self.CONSUMER_GRACE_TIME, result[ 'Value' ] )
    return self.__cleanReturn( result )

  def getConsumerData( self, consumerKey ):
    cData = self.__consumers.get( consumerKey )
    if cData:
      return S_OK( cData )
    result = self.__getRPC().getConsumerData( consumerKey )
    if not result[ 'OK' ]:
      return self.__cleanReturn( result )
    self.__consumers.add( consumerKey, self.CONSUMER_GRACE_TIME, result[ 'Value' ] )
    return self.__cleanReturn( result )

  def deleteConsumer( self, consumerKey ):
    self.__consumers.delete( consumerKey )
    result = self.__getRPC().deleteConsumer( consumerKey )
    if result[ 'OK' ]:
      self.__cleanConsumerCache( { 'key' : consumerKey } )
    return self.__cleanReturn( result )

  def getAllConsumers( self ):
    result = self.__getRPC().getAllConsumers()
    if not result[ 'OK' ]:
      return self.__cleanReturn( result )
    data = result[ 'Value' ]
    consIndex = { 'key': 0,
                  'name' : 0,
                  'callback' : 0,
                  'secret' : 0,
                  'icon' : 0 }
    for key in consIndex:
      consIndex[ key ] = data[ 'Parameters' ].find( key )
    for record in data[ 'Records' ]:
      consData = {}
      for key in consIndex:
        consData[ key ] = record[ consIndex[ key ] ]
      self.__consumers.add( consData[ 'key' ], self.CONSUMER_GRACE_TIME, consData )
    return self.__cleanReturn( result )

  def __cleanConsumerCache( self, cData ):
    consumerKey = cData[ 'key' ]
    for dc in ( self.__tokens, self.__requests ):
      cKeys = dc.getKeys()
      for cKey in cKeys:
        if cKey[0] == consumerKey:
          dc.delete( cKey )

  ##
  # Requests
  ##

  def generateRequest( self, consumerKey, callback = "" ):
    result = self.__getRPC().generateRequest( consumerKey, callback )
    if not result[ 'OK' ]:
      return self.__cleanReturn( result )
    requestData = result[ 'Value' ]
    self.__requests.add( requestData[ 'request' ], result[ 'lifeTime' ] - 5, requestData )
    return self.__cleanReturn( result )

  def getRequestData( self, request ):
    data = self.__requests.get( request )
    if data:
      return S_OK( data )
    result = self.__getRPC().getRequestData( request )
    if not result[ 'OK' ]:
      return self.__cleanReturn( result )
    self.__tokens.add( request, result[ 'lifeTime' ] - 5, result[ 'Value' ] )
    return self.__cleanReturn( result )

  def deleteRequest( self, request ):
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.DictCache import DictCache [as 别名]
# 或者: from DIRAC.Core.Utilities.DictCache.DictCache import getKeys [as 别名]
class GridPilotDirector( PilotDirector ):
  """
    Base Grid PilotDirector class
    Derived classes must declare:
      self.Middleware: It must correspond to the string before "PilotDirector".
        (For proper naming of the logger)
      self.ResourceBrokers: list of Brokers used by the Director.
        (For proper error reporting)
  """
  def __init__( self, submitPool ):
    """
     Define some defaults and call parent __init__
    """
    self.gridEnv = GRIDENV

    self.cpuPowerRef = CPU_POWER_REF
    self.requirements = REQUIREMENTS
    self.rank = RANK
    self.fuzzyRank = FUZZY_RANK

    self.__failingWMSCache = DictCache()
    self.__ticketsWMSCache = DictCache()
    self.__listMatchWMSCache = DictCache()

    PilotDirector.__init__( self, submitPool )

  def configure( self, csSection, submitPool ):
    """
     Here goes common configuration for all Grid PilotDirectors
    """
    PilotDirector.configure( self, csSection, submitPool )
    self.reloadConfiguration( csSection, submitPool )

    self.__failingWMSCache.purgeExpired()
    self.__ticketsWMSCache.purgeExpired()
    for rb in self.__failingWMSCache.getKeys():
      if rb in self.resourceBrokers:
        try:
          self.resourceBrokers.remove( rb )
        except:
          pass

    self.resourceBrokers = List.randomize( self.resourceBrokers )

    if self.gridEnv:
      self.log.info( ' GridEnv:        ', self.gridEnv )
    if self.resourceBrokers:
      self.log.info( ' ResourceBrokers:', ', '.join( self.resourceBrokers ) )

  def configureFromSection( self, mySection ):
    """
      reload from CS
    """
    PilotDirector.configureFromSection( self, mySection )

    self.gridEnv = gConfig.getValue( mySection + '/GridEnv', self.gridEnv )
    if not self.gridEnv:
      # No specific option found, try a general one
      setup = gConfig.getValue( '/DIRAC/Setup', '' )
      if setup:
        instance = gConfig.getValue( '/DIRAC/Setups/%s/WorkloadManagement' % setup, '' )
        if instance:
          self.gridEnv = gConfig.getValue( '/Systems/WorkloadManagement/%s/GridEnv' % instance, '' )

    self.resourceBrokers = gConfig.getValue( mySection + '/ResourceBrokers'      , self.resourceBrokers )

    self.cpuPowerRef = gConfig.getValue( mySection + '/CPUPowerRef'           , self.cpuPowerRef )
    self.requirements = gConfig.getValue( mySection + '/Requirements'         , self.requirements )
    self.rank = gConfig.getValue( mySection + '/Rank'                 , self.rank )
    self.fuzzyRank = gConfig.getValue( mySection + '/FuzzyRank'            , self.fuzzyRank )

  def _submitPilots( self, workDir, taskQueueDict, pilotOptions, pilotsToSubmit,
                     ceMask, submitPrivatePilot, privateTQ, proxy, pilotsPerJob ):
    """
      This method does the actual pilot submission to the Grid RB
      The logic is as follows:
      - If there are no available RB it return error
      - If there is no VOMS extension in the proxy, return error
      - It creates a temp directory
      - Prepare a JDL
        it has some part common to gLite and LCG (the payload description)
        it has some part specific to each middleware
    """
    taskQueueID = taskQueueDict['TaskQueueID']
    # ownerDN = taskQueueDict['OwnerDN']
    credDict = proxy.getCredentials()['Value']
    ownerDN = credDict['identity']
    ownerGroup = credDict[ 'group' ]

    if not self.resourceBrokers:
      # Since we can exclude RBs from the list, it may become empty
      return S_ERROR( ERROR_RB )

    # Need to get VOMS extension for the later interactions with WMS
    ret = gProxyManager.getVOMSAttributes( proxy )
    if not ret['OK']:
      self.log.error( ERROR_VOMS, ret['Message'] )
      return S_ERROR( ERROR_VOMS )
    if not ret['Value']:
      return S_ERROR( ERROR_VOMS )
#.........这里部分代码省略.........