Python DictCache.DictCache类代码示例

class StorageElementCache(object):

  def __init__(self):
    self.seCache = DictCache()

  def __call__(self, name, plugins=None, vo=None, hideExceptions=False):
    self.seCache.purgeExpired(expiredInSeconds=60)
    tId = threading.current_thread().ident

    if not vo:
      result = getVOfromProxyGroup()
      if not result['OK']:
        return
      vo = result['Value']

    # Because the gfal2 context caches the proxy location,
    # we also use the proxy location as a key.
    # In practice, there should almost always be one, except for the REA
    # If we see its memory consumtpion exploding, this might be a place to look
    proxyLoc = getProxyLocation()

    argTuple = (tId, name, plugins, vo, proxyLoc)
    seObj = self.seCache.get(argTuple)

    if not seObj:
      seObj = StorageElementItem(name, plugins, vo, hideExceptions=hideExceptions)
      # Add the StorageElement to the cache for 1/2 hour
      self.seCache.add(argTuple, 1800, seObj)

    return seObj

 def __init__( self, RPCFunctor = None ):
   if not RPCFunctor:
     self.__RPCFunctor = RPCClient
   else:
     self.__RPCFunctor = RPCFunctor
   self.__tokens = DictCache()
   self.__requests = DictCache()
   self.__consumers = DictCache( deleteFunction = self.__cleanConsumerCache )

 def __init__( self ):
   self.graphsLocation = os.path.join( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), 'data', 'accountingPlots' )
   self.cachedGraphs = {}
   self.alive = True
   self.purgeThread = threading.Thread( target = self.purgeExpired )
   self.purgeThread.setDaemon( 1 )
   self.purgeThread.start()
   self.__dataCache = DictCache()
   self.__graphCache = DictCache( deleteFunction = self._deleteGraph )
   self.__dataLifeTime = 600
   self.__graphLifeTime = 3600

class PlotCache:

  def __init__( self, plotsLocation = False ):
    self.plotsLocation = plotsLocation
    self.alive = True
    self.__graphCache = DictCache( deleteFunction = _deleteGraph )
    self.__graphLifeTime = 600
    self.purgeThread = threading.Thread( target = self.purgeExpired )
    self.purgeThread.setDaemon( 1 )
    self.purgeThread.start()

  def setPlotsLocation( self, plotsDir ):
    self.plotsLocation = plotsDir
    for plot in os.listdir( self.plotsLocation ):
      if plot.find( ".png" ) > 0:
        plotLocation = "%s/%s" % ( self.plotsLocation, plot )
        gLogger.verbose( "Purging %s" % plotLocation )
        os.unlink( plotLocation )

  def purgeExpired( self ):
    while self.alive:
      time.sleep( self.__graphLifeTime )
      self.__graphCache.purgeExpired()

  def getPlot( self, plotHash, plotData, plotMetadata, subplotMetadata ):
    """
    Get plot from the cache if exists, else generate it
    """

    plotDict = self.__graphCache.get( plotHash )
    if plotDict == False:
      basePlotFileName = "%s/%s.png" % ( self.plotsLocation, plotHash )
      if subplotMetadata:
        retVal = graph( plotData, basePlotFileName, plotMetadata, metadata = subplotMetadata )
      else:
        retVal = graph( plotData, basePlotFileName, plotMetadata )
      if not retVal[ 'OK' ]:
        return retVal
      plotDict = retVal[ 'Value' ]
      if plotDict[ 'plot' ]:
        plotDict[ 'plot' ] = os.path.basename( basePlotFileName )
      self.__graphCache.add( plotHash, self.__graphLifeTime, plotDict )
    return S_OK( plotDict )

  def getPlotData( self, plotFileName ):
    filename = "%s/%s" % ( self.plotsLocation, plotFileName )
    try:
      fd = file( filename, "rb" )
      data = fd.read()
      fd.close()
    except Exception, v:
      return S_ERROR( "Can't open file %s: %s" % ( plotFileName, str( v ) ) )
    return S_OK( data )

class StorageElementCache(object):
    def __init__(self):
        self.seCache = DictCache()

    def __call__(self, name, protocols=None, vo=None, hideExceptions=False):
        self.seCache.purgeExpired(expiredInSeconds=60)
        argTuple = (name, protocols, vo)
        seObj = self.seCache.get(argTuple)

        if not seObj:
            seObj = StorageElementItem(name, protocols, vo, hideExceptions=hideExceptions)
            # Add the StorageElement to the cache for 1/2 hour
            self.seCache.add(argTuple, 1800, seObj)

        return seObj

  def __init__( self, submitPool ):
    """
     Define some defaults and call parent __init__
    """
    self.gridEnv = GRIDENV

    self.cpuPowerRef = CPU_POWER_REF
    self.requirements = REQUIREMENTS
    self.rank = RANK
    self.fuzzyRank = FUZZY_RANK

    self.__failingWMSCache = DictCache()
    self.__ticketsWMSCache = DictCache()
    self.__listMatchWMSCache = DictCache()

    PilotDirector.__init__( self, submitPool )

 def __init__( self, lifeTime, updateFunc ):
   """
   Constructor
   
   :Parameters:
     **lifeTime** - `int`
       Lifetime of the elements in the cache ( seconds ! )
     **updateFunc** - `function`
       This function MUST return a S_OK | S_ERROR object. In the case of the first,
       its value must be a dictionary.
   
   """
   
   # We set a 20% of the lifetime randomly, so that if we have thousands of jobs
   # starting at the same time, all the caches will not end at the same time.
   randomLifeTimeBias  = 0.2 * random.random()
   
   self.log            = gLogger.getSubLogger( self.__class__.__name__ )
   
   self.__lifeTime     = int( lifeTime * ( 1 + randomLifeTimeBias ) )
   self.__updateFunc   = updateFunc
   # The records returned from the cache must be valid at least 10 seconds.
   self.__validSeconds = 10
   
   # Cache
   self.__cache       = DictCache()
   self.__cacheLock   = LockRing()
   self.__cacheLock.getLock( self.__class__.__name__ )

 def __init__( self, plotsLocation = False ):
   self.plotsLocation = plotsLocation
   self.alive = True
   self.__graphCache = DictCache( deleteFunction = _deleteGraph )
   self.__graphLifeTime = 600
   self.purgeThread = threading.Thread( target = self.purgeExpired )
   self.purgeThread.start()

  def __init__( self, submitPool ):
    """
     Define some defaults and call parent __init__
    """
    self.gridMiddleware    = 'DIRAC'

    PilotDirector.__init__( self, submitPool )

    self.computingElementList = COMPUTING_ELEMENTS
    self.computingElementDict = {}
    self.addComputingElement( self.computingElementList )

    self.siteName          = gConfig.getValue('/LocalSite/Site','')
    if not self.siteName:
      self.log.error( 'Can not run a Director if Site Name is not defined' )
      sys.exit()

    self.__failingCECache  = DictCache()
    self.__ticketsCECache  = DictCache()

  def __init__(self, jobDB=None, opsHelper=None):
    """ Constructor
    """
    self.__runningLimitSection = "JobScheduling/RunningLimit"
    self.__matchingDelaySection = "JobScheduling/MatchingDelay"
    self.csDictCache = DictCache()
    self.condCache = DictCache()
    self.delayMem = {}

    if jobDB:
      self.jobDB = jobDB
    else:
      self.jobDB = JobDB()

    self.log = gLogger.getSubLogger("Limiter")

    if opsHelper:
      self.__opsHelper = opsHelper
    else:
      self.__opsHelper = Operations()

 def __init__( self ):
   """ Initialize like a real service
   """
   super(GatewayService, self).__init__(
       {'modName':GatewayService.GATEWAY_NAME,
        'loadName':GatewayService.GATEWAY_NAME,
        'standalone': True,
        'moduleObj': sys.modules[DIRAC.Core.DISET.private.GatewayService.GatewayService.__module__],
        'classObj': self.__class__} )
   self.__delegatedCredentials = DictCache()
   self.__transferBytesLimit = 1024 * 1024 * 100
   # to be resolved
   self._url = None
   self._handler = None
   self._threadPool = None
   self._msgBroker = None
   self._msgForwarder = None

  def __init__( self, lifeTime, updateFunc = None, cacheHistoryLifeTime = None ):
    '''
    Constructor
    '''

    self.__lifeTime             = lifeTime
    # lifetime of the history on hours
    self.__cacheHistoryLifeTime = ( 1 and cacheHistoryLifeTime ) or 24
    self.__updateFunc           = updateFunc

    # RSSCache
    self.__rssCache       = DictCache()
    self.__rssCacheStatus = [] # ( updateTime, message )
    self.__rssCacheLock   = threading.Lock()

    # Create purgeThread
    self.__refreshStop    = False
    self.__refreshThread  = threading.Thread( target = self.__refreshCacheThreadRun )
    self.__refreshThread.setDaemon( True )

  def __init__( self, submitPool ):
    """
     Define the logger and some defaults
    """

    if submitPool == self.gridMiddleware:
      self.log = gLogger.getSubLogger( '%sPilotDirector' % self.gridMiddleware )
    else:
      self.log = gLogger.getSubLogger( '%sPilotDirector/%s' % ( self.gridMiddleware, submitPool ) )

    self.pilot = DIRAC_PILOT
    self.submitPoolOption = '-o /Resources/Computing/CEDefaults/SubmitPool=%s' % submitPool
    self.extraPilotOptions = []
    self.installVersion = DIRAC_VERSION
    self.installProject = DIRAC_PROJECT
    self.installation = DIRAC_INSTALLATION
    self.pilotExtensionsList = []

    self.virtualOrganization = VIRTUAL_ORGANIZATION
    self.install = DIRAC_INSTALL
    self.extraModules = DIRAC_MODULES
    self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
    self.targetGrids = [ self.gridMiddleware ]


    self.enableListMatch = ENABLE_LISTMATCH
    self.listMatchDelay = LISTMATCH_DELAY
    self.listMatchCache = DictCache()

    self.privatePilotFraction = PRIVATE_PILOT_FRACTION

    self.errorClearTime = ERROR_CLEAR_TIME
    self.errorTicketTime = ERROR_TICKET_TIME
    self.errorMailAddress = DIRAC.errorMail
    self.alarmMailAddress = DIRAC.alarmMail
    self.mailFromAddress = FROM_MAIL

    if not  'log' in self.__dict__:
      self.log = gLogger.getSubLogger( 'PilotDirector' )
    self.log.info( 'Initialized' )

class Limiter(object):

  def __init__(self, jobDB=None, opsHelper=None):
    """ Constructor
    """
    self.__runningLimitSection = "JobScheduling/RunningLimit"
    self.__matchingDelaySection = "JobScheduling/MatchingDelay"
    self.csDictCache = DictCache()
    self.condCache = DictCache()
    self.delayMem = {}

    if jobDB:
      self.jobDB = jobDB
    else:
      self.jobDB = JobDB()

    self.log = gLogger.getSubLogger("Limiter")

    if opsHelper:
      self.__opsHelper = opsHelper
    else:
      self.__opsHelper = Operations()

  def getNegativeCond(self):
    """ Get negative condition for ALL sites
    """
    orCond = self.condCache.get("GLOBAL")
    if orCond:
      return orCond
    negCond = {}
    # Run Limit
    result = self.__opsHelper.getSections(self.__runningLimitSection)
    sites = []
    if result['OK']:
      sites = result['Value']
    for siteName in sites:
      result = self.__getRunningCondition(siteName)
      if not result['OK']:
        continue
      data = result['Value']
      if data:
        negCond[siteName] = data
    # Delay limit
    result = self.__opsHelper.getSections(self.__matchingDelaySection)
    sites = []
    if result['OK']:
      sites = result['Value']
    for siteName in sites:
      result = self.__getDelayCondition(siteName)
      if not result['OK']:
        continue
      data = result['Value']
      if not data:
        continue
      if siteName in negCond:
        negCond[siteName] = self.__mergeCond(negCond[siteName], data)
      else:
        negCond[siteName] = data
    orCond = []
    for siteName in negCond:
      negCond[siteName]['Site'] = siteName
      orCond.append(negCond[siteName])
    self.condCache.add("GLOBAL", 10, orCond)
    return orCond

  def getNegativeCondForSite(self, siteName):
    """ Generate a negative query based on the limits set on the site
    """
    # Check if Limits are imposed onto the site
    negativeCond = {}
    if self.__opsHelper.getValue("JobScheduling/CheckJobLimits", True):
      result = self.__getRunningCondition(siteName)
      if result['OK']:
        negativeCond = result['Value']
      self.log.verbose('Negative conditions for site %s after checking limits are: %s' % (siteName, str(negativeCond)))

    if self.__opsHelper.getValue("JobScheduling/CheckMatchingDelay", True):
      result = self.__getDelayCondition(siteName)
      if result['OK']:
        delayCond = result['Value']
        self.log.verbose('Negative conditions for site %s after delay checking are: %s' % (siteName, str(delayCond)))
        negativeCond = self.__mergeCond(negativeCond, delayCond)

    if negativeCond:
      self.log.info('Negative conditions for site %s are: %s' % (siteName, str(negativeCond)))

    return negativeCond

  def __mergeCond(self, negCond, addCond):
    """ Merge two negative dicts
    """
    # Merge both negative dicts
    for attr in addCond:
      if attr not in negCond:
        negCond[attr] = []
      for value in addCond[attr]:
        if value not in negCond[attr]:
          negCond[attr].append(value)
    return negCond

#.........这里部分代码省略.........

class RSSCache( object ):
  '''
  Cache with purgeThread integrated
  '''

  def __init__( self, lifeTime, updateFunc = None, cacheHistoryLifeTime = None ):
    '''
    Constructor
    '''

    self.__lifeTime             = lifeTime
    # lifetime of the history on hours
    self.__cacheHistoryLifeTime = ( 1 and cacheHistoryLifeTime ) or 24
    self.__updateFunc           = updateFunc

    # RSSCache
    self.__rssCache       = DictCache()
    self.__rssCacheStatus = [] # ( updateTime, message )
    self.__rssCacheLock   = threading.Lock()

    # Create purgeThread
    self.__refreshStop    = False
    self.__refreshThread  = threading.Thread( target = self.__refreshCacheThreadRun )
    self.__refreshThread.setDaemon( True )

  def startRefreshThread( self ):
    '''
    Run refresh thread.
    '''
    self.__refreshThread.start()

  def stopRefreshThread( self ):
    '''
    Stop refresh thread.
    '''
    self.__refreshStop = True

  def isCacheAlive( self ):
    '''
    Returns status of the cache refreshing thread
    '''
    return S_OK( self.__refreshThread.isAlive() )

  def setLifeTime( self, lifeTime ):
    '''
    Set cache life time
    '''
    self.__lifeTime = lifeTime

  def setCacheHistoryLifeTime( self, cacheHistoryLifeTime ):
    '''
    Set cache life time
    '''
    self.__cacheHistoryLifeTime = cacheHistoryLifeTime

  def getCacheKeys( self ):
    '''
    List all the keys stored in the cache.
    '''
    self.__rssCacheLock.acquire()
    keys = self.__rssCache.getKeys()
    self.__rssCacheLock.release()

    return S_OK( keys )

  def acquireLock( self ):
    '''
    Acquires RSSCache lock
    '''
    self.__rssCacheLock.acquire()

  def releaseLock( self ):
    '''
    Releases RSSCache lock
    '''
    self.__rssCacheLock.release()

  def getCacheStatus( self ):
    '''
    Return the latest cache status
    '''
    self.__rssCacheLock.acquire()
    if self.__rssCacheStatus:
      res = dict( [ self.__rssCacheStatus[ 0 ] ] )
    else:
      res = {}
    self.__rssCacheLock.release()
    return S_OK( res )

  def getCacheHistory( self ):
    '''
    Return the cache updates history
    '''
    self.__rssCacheLock.acquire()
    res = dict( self.__rssCacheStatus )
    self.__rssCacheLock.release()
    return S_OK( res )

  def get( self, resourceKey ):
    '''
#.........这里部分代码省略.........