Python LockRing.LockRing类代码示例

 def __init__( self, lifeTime, updateFunc ):
   """
   Constructor
   
   :Parameters:
     **lifeTime** - `int`
       Lifetime of the elements in the cache ( seconds ! )
     **updateFunc** - `function`
       This function MUST return a S_OK | S_ERROR object. In the case of the first,
       its value must be a dictionary.
   
   """
   
   # We set a 20% of the lifetime randomly, so that if we have thousands of jobs
   # starting at the same time, all the caches will not end at the same time.
   randomLifeTimeBias  = 0.2 * random.random()
   
   self.log            = gLogger.getSubLogger( self.__class__.__name__ )
   
   self.__lifeTime     = int( lifeTime * ( 1 + randomLifeTimeBias ) )
   self.__updateFunc   = updateFunc
   # The records returned from the cache must be valid at least 10 seconds.
   self.__validSeconds = 10
   
   # Cache
   self.__cache       = DictCache()
   self.__cacheLock   = LockRing()
   self.__cacheLock.getLock( self.__class__.__name__ )

class Synchronizer:
    """ Class encapsulating a lock
  allowing it to be used as a synchronizing
  decorator making the call thread-safe"""

    def __init__(self, lockName="", recursive=False):
        from DIRAC.Core.Utilities.LockRing import LockRing

        self.__lockName = lockName
        self.__lr = LockRing()
        self.__lock = self.__lr.getLock(lockName, recursive=recursive)

    def __call__(self, funcToCall):
        def lockedFunc(*args, **kwargs):
            try:
                if self.__lockName:
                    print "LOCKING", self.__lockName
                self.__lock.acquire()
                return funcToCall(*args, **kwargs)
            finally:
                if self.__lockName:
                    print "UNLOCKING", self.__lockName
                self.__lock.release()

        return lockedFunc

    def lock(self):
        return self.__lock.acquire()

    def unlock(self):
        return self.__lock.release()

 def __init__( self, deleteFunction = False ):
   """
   Initialize the dict cache.
     If a delete function is specified it will be invoked when deleting a cached object
   """
   
   self.__lock = LockRing()
   self.__lock.getLock( self.__class__.__name__, recursive = True )
   
   self.__cache = {}
   self.__deleteFunction = deleteFunction

 def __init__( self, loadDefaultCFG = True ):
   lr = LockRing()
   self.threadingEvent = lr.getEvent()
   self.threadingEvent.set()
   self.threadingLock = lr.getLock()
   self.runningThreadsNumber = 0
   self.compressedConfigurationData = ""
   self.configurationPath = "/DIRAC/Configuration"
   self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
   self._isService = False
   self.localCFG = CFG()
   self.remoteCFG = CFG()
   self.mergedCFG = CFG()
   self.remoteServerList = []
   if loadDefaultCFG:
     defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
     gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
     retVal = self.loadFile( defaultCFGFile )
     if not retVal[ 'OK' ]:
       gLogger.warn( "Can't load %s file" % defaultCFGFile )
   self.sync()

  def wrapped_fcn( *args, **kwargs ):

    # Get the lock and acquire it
    executionLock = LockRing().getLock( '_UseUserProxy_', recursive = True )
    executionLock.acquire()

    # Check if the caller is executing with the host certificate
    useServerCertificate = gConfig.useServerCertificate()
    if useServerCertificate:
      gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )

    try:
      return fcn( *args, **kwargs )
    except Exception as lException:  # pylint: disable=broad-except
      value = ','.join( [str( arg ) for arg in lException.args] )
      exceptType = lException.__class__.__name__
      return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
    finally:
      # Restore the default host certificate usage if necessary
      if useServerCertificate:
        gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
      # release the lock
      executionLock.release()

def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFilePath=None, executionLockFlag=False):
  """Download proxy, place in a file and populate X509_USER_PROXY environment variable.

  Parameters like `userProxy` or `executeWithUserProxy`.
  :returns: Tuple of originalUserProxy, useServerCertificate, executionLock
  """
  # Setup user proxy
  if userDN:
    userDNs = [userDN]
  else:
    result = getDNForUsername(userName)
    if not result['OK']:
      return result
    userDNs = result['Value']  # a same user may have more than one DN

  vomsAttr = ''
  if vomsFlag:
    vomsAttr = getVOMSAttributeForGroup(userGroup)

  result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath)

  if not result['OK']:
    return result

  executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) if executionLockFlag else None
  if executionLockFlag:
    executionLock.acquire()

  os.environ['X509_USER_PROXY'], originalUserProxy = result['Value'], os.environ.get('X509_USER_PROXY')

  # Check if the caller is executing with the host certificate
  useServerCertificate = gConfig.useServerCertificate()
  if useServerCertificate:
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')

  return S_OK((originalUserProxy, useServerCertificate, executionLock))

  def run( self ):
    """ task execution

    reads and executes ProcessTask :task: out of pending queue and then pushes it
    to the results queue for callback execution

    :param self: self reference
    """
    ## start watchdog thread
    print "PID:%s PPID:%s Started WorkingProcess...." % (os.getpid(), os.getppid()) 
    self.__watchdogThread = threading.Thread( target = self.__watchdog )
    self.__watchdogThread.daemon = True
    self.__watchdogThread.start()
    print "PID:%s PPID:%s started watchdogThread.." % (os.getpid(), os.getppid())
    ## http://cdn.memegenerator.net/instances/400x/19450565.jpg
    if LockRing:
      # Reset all locks
      lr = LockRing()
      lr._openAll()
      lr._setAllEvents()

    ## zero processed task counter
    taskCounter = 0
    ## zero idle loop counter
    idleLoopCount = 0

    ## main loop
    while True:

      ## draining, stopEvent is set, exiting 
      if self.__stopEvent.is_set():
        return

      ## clear task
      self.task = None

      ## read from queue
      try:
	print "PID:%s PPID:%s Will try to GET from Pending Queue..." % (os.getpid(), os.getppid())
        task = self.__pendingQueue.get( block = True, timeout = 5 )   # timeout changed from 10
        print "PID:%s PPID:%s GOT a task from pending queue.." % (os.getpid(), os.getppid())
      except Queue.Empty:
        ## idle loop?
        idleLoopCount += 1
        ## 10th idle loop - exit, nothing to do 
        if idleLoopCount == 2: # changed from 10
          print "PID:%s PPID:%s Tried 10 times to get something, exiting..."% (os.getpid(), os.getppid())
          return 
        continue

      ## toggle __working flag
      self.__working.value = 1
      ## save task
      self.task = task
      ## reset idle loop counter
      idleLoopCount = 0

      ## process task in a separate thread
      self.__processThread = threading.Thread( target = self.__processTask )
      self.__processThread.start()
      print "PID:%s PPID:%s started process Thread..." % (os.getpid(), os.getppid())
      ## join processThread with or without timeout
      if self.task.getTimeOut():
	print "WILL WAIT FOR JOIN(timeout)"
        self.__processThread.join( self.task.getTimeOut()+10 )
      else:
	print "WILL WAIT FOR JOIN"
        self.__processThread.join()

      ## processThread is still alive? stop it!
      if self.__processThread.is_alive():
        self.__processThread._Thread__stop()
        print "MUST FORCE-STOP PROCESS THREAD"
      
      print "PID:%s PPID:%s Process thread done..."% (os.getpid(), os.getppid())
      ## check results and callbacks presence, put task to results queue
      if self.task.hasCallback() or self.task.hasPoolCallback():
        if not self.task.taskResults() and not self.task.taskException():
          self.task.setResult( S_ERROR("Timed out") )
          print ">>>TIMED OUT!!!!"
        self.__resultsQueue.put( task )
        print "ResultsQueue = %s " % self.__resultsQueue.qsize()
      ## increase task counter
      taskCounter += 1
      self.__taskCounter = taskCounter 
      ## toggle __working flag
      self.__working.value = 0

  def run( self ):
    """ task execution

    reads and executes ProcessTask :task: out of pending queue and then pushes it
    to the results queue for callback execution

    :param self: self reference
    """
    ## start watchdog thread
    self.__watchdogThread = threading.Thread( target = self.__watchdog )
    self.__watchdogThread.daemon = True
    self.__watchdogThread.start()

    ## http://cdn.memegenerator.net/instances/400x/19450565.jpg
    if LockRing:
      # Reset all locks
      lr = LockRing()
      lr._openAll()
      lr._setAllEvents()

    ## zero processed task counter
    taskCounter = 0
    ## zero idle loop counter
    idleLoopCount = 0

    ## main loop
    while True:

      ## draining, stopEvent is set, exiting 
      if self.__stopEvent.is_set():
        return

      ## clear task
      self.task = None

      ## read from queue
      try:
        task = self.__pendingQueue.get( block = True, timeout = 10 )   
      except Queue.Empty:
        ## idle loop?
        idleLoopCount += 1
        ## 10th idle loop - exit, nothing to do 
        if idleLoopCount == 10:
          return 
        continue

      ## toggle __working flag
      self.__working.value = 1
      ## save task
      self.task = task
      ## reset idle loop counter
      idleLoopCount = 0

      ## process task in a separate thread
      self.__processThread = threading.Thread( target = self.__processTask )
      self.__processThread.start()

      ## join processThread with or without timeout
      if self.task.getTimeOut():
        self.__processThread.join( self.task.getTimeOut()+10 )
      else:
        self.__processThread.join()

      ## processThread is still alive? stop it!
      if self.__processThread.is_alive():
        self.__processThread._Thread__stop()
      
      ## check results and callbacks presence, put task to results queue
      if self.task.hasCallback() or self.task.hasPoolCallback():
        if not self.task.taskResults() and not self.task.taskException():
          self.task.setResult( S_ERROR("Timed out") )
        self.__resultsQueue.put( task )
      ## increase task counter
      taskCounter += 1
      self.__taskCounter = taskCounter 
      ## toggle __working flag
      self.__working.value = 0

  def run( self ):
    """ 
    Task execution

    Reads and executes ProcessTask :task: out of pending queue and then pushes it
    to the results queue for callback execution.

    :param self: self reference
    """
    ## start watchdog thread
    self.__watchdogThread = threading.Thread( target = self.__watchdog )
    self.__watchdogThread.daemon = True
    self.__watchdogThread.start()

    ## http://cdn.memegenerator.net/instances/400x/19450565.jpg
    if LockRing:
      # Reset all locks
      lr = LockRing()
      lr._openAll()
      lr._setAllEvents()

    ## zero processed task counter
    taskCounter = 0
    ## zero idle loop counter
    idleLoopCount = 0

    ## main loop
    while True:

      ## draining, stopEvent is set, exiting 
      if self.__stopEvent.is_set():
        return

      ## clear task
      self.task = None

      ## read from queue
      try:
        task = self.__pendingQueue.get( block = True, timeout = 10 )   
      except Queue.Empty:
        ## idle loop?
        idleLoopCount += 1
        ## 10th idle loop - exit, nothing to do 
        if idleLoopCount == 10 and not self.__keepRunning:
          return 
        continue

      ## toggle __working flag
      self.__working.value = 1
      ## save task
      self.task = task
      ## reset idle loop counter
      idleLoopCount = 0

      ## process task in a separate thread
      self.__processThread = threading.Thread( target = self.__processTask )
      self.__processThread.start()

      timeout = False
      noResults = False
      ## join processThread with or without timeout
      if self.task.getTimeOut():
        self.__processThread.join( self.task.getTimeOut()+10 )
      else:
        self.__processThread.join()

      ## processThread is still alive? stop it!
      if self.__processThread.is_alive():
        self.__processThread._Thread__stop()
        self.task.setResult( S_ERROR( errno.ETIME, "Timed out" ) )
        timeout = True
      # if the task finished with no results, something bad happened, e.g. 
      # undetected timeout  
      if not self.task.taskResults() and not self.task.taskException():
        self.task.setResult( S_ERROR("Task produced no results") )  
        noResults = True
      
      ## check results and callbacks presence, put task to results queue
      if self.task.hasCallback() or self.task.hasPoolCallback():
        self.__resultsQueue.put( task )
      if timeout or noResults:  
        # The task execution timed out, stop the process to prevent it from running 
        # in the background
        time.sleep( 1 )
        os.kill( self.pid, signal.SIGKILL )
        return   
      ## increase task counter
      taskCounter += 1
      self.__taskCounter = taskCounter 
      ## toggle __working flag
      self.__working.value = 0

 def __init__( self, msgBroker ):
   self.__inOutLock = LockRing().getLock()
   self.__msgBroker = msgBroker
   self.__byClient = {}
   self.__srvToCliTrid = {}

class MessageForwarder(object):

  def __init__( self, msgBroker ):
    self.__inOutLock = LockRing().getLock()
    self.__msgBroker = msgBroker
    self.__byClient = {}
    self.__srvToCliTrid = {}

  def addClient( self, cliTrid, destination, clientInitParams, connectParams ):
    if cliTrid in self.__byClient:
      gLogger.fatal( "Trid is duplicated!! this shouldn't happen" )
      return
    msgClient = MessageClient( destination, **clientInitParams )
    msgClient.subscribeToDisconnect( self.__srvDisconnect )
    msgClient.subscribeToAllMessages( self.msgFromSrv )
    msgClient.setUniqueName( connectParams[0] )
    result = msgClient.connect( **connectParams[1] )
    if not result[ 'OK' ]:
      return result
    self.__inOutLock.acquire()
    try:
      self.__byClient[ cliTrid ] = { 'srvEnd' : msgClient,
                                     'srvTrid' : msgClient.getTrid(),
                                     'srvName' : destination }
      self.__srvToCliTrid[ msgClient.getTrid() ] = cliTrid
    finally:
      self.__inOutLock.release()
    return result

  def __srvDisconnect( self, srvEndCli ):
    try:
      cliTrid = self.__srvToCliTrid[ srvEndCli.getTrid() ]
    except IndexError:
      gLogger.exception( "This shouldn't happen!" )
    gLogger.info( "Service %s disconnected messaging connection" % self.__byClient[ cliTrid ][ 'srvName' ] )
    self.__msgBroker.removeTransport( cliTrid )
    self.__removeClient( cliTrid )

  def cliDisconnect( self, cliTrid ):
    if cliTrid not in self.__byClient:
      gLogger.fatal( "This shouldn't happen!" )
      return
    gLogger.info( "Client to %s disconnected messaging connection" % self.__byClient[ cliTrid ][ 'srvName' ] )
    self.__byClient[ cliTrid ][ 'srvEnd' ].disconnect()
    self.__removeClient( cliTrid )

  def __removeClient( self, cliTrid ):
    self.__inOutLock.acquire()
    try:
      try:
        srvTrid = self.__byClient[ cliTrid ][ 'srvTrid' ]
        self.__byClient.pop( cliTrid )
        self.__srvToCliTrid.pop( srvTrid )
      except Exception as e:
        gLogger.exception( "This shouldn't happen!" )
    finally:
      self.__inOutLock.release()

  def msgFromClient( self, cliTrid, msgObj ):
    gLogger.info( "Message %s to %s service" % ( msgObj.getName(), self.__byClient[ cliTrid ][ 'srvName' ] ) )
    result = self.__byClient[ cliTrid ][ 'srvEnd' ].sendMessage( msgObj )
    return result

  def msgFromSrv( self, srvEndCli, msgObj ):
    try:
      cliTrid = self.__srvToCliTrid[ srvEndCli.getTrid() ]
    except:
      gLogger.exception( "This shouldn't happen" )
      return S_ERROR( "MsgFromSrv -> Mismatched srv2cli trid" )
    gLogger.info( "Message %s from %s service" % ( msgObj.getName(), self.__byClient[ cliTrid ][ 'srvName' ] ) )
    return self.__msgBroker.sendMessage( cliTrid, msgObj )

class Cache( object ):
  """
    Cache basic class.
    
    WARNING: None of its methods is thread safe. Acquire / Release lock when
    using them !
  """
  
  def __init__( self, lifeTime, updateFunc ):
    """
    Constructor
    
    :Parameters:
      **lifeTime** - `int`
        Lifetime of the elements in the cache ( seconds ! )
      **updateFunc** - `function`
        This function MUST return a S_OK | S_ERROR object. In the case of the first,
        its value must be a dictionary.
    
    """
    
    # We set a 20% of the lifetime randomly, so that if we have thousands of jobs
    # starting at the same time, all the caches will not end at the same time.
    randomLifeTimeBias  = 0.2 * random.random()
    
    self.log            = gLogger.getSubLogger( self.__class__.__name__ )
    
    self.__lifeTime     = int( lifeTime * ( 1 + randomLifeTimeBias ) )
    self.__updateFunc   = updateFunc
    # The records returned from the cache must be valid at least 10 seconds.
    self.__validSeconds = 10
    
    # Cache
    self.__cache       = DictCache()
    self.__cacheLock   = LockRing()
    self.__cacheLock.getLock( self.__class__.__name__ )
  
  #.............................................................................
  # internal cache object getter
  
  def cacheKeys( self ):
    """
    Cache keys getter
      
    :returns: list with valid keys on the cache
    """
    
    return self.__cache.getKeys( validSeconds = self.__validSeconds ) 

  #.............................................................................
  # acquire / release Locks

  def acquireLock( self ):
    """
    Acquires Cache lock
    """
    
    self.__cacheLock.acquire( self.__class__.__name__ )

  def releaseLock( self ):
    """
    Releases Cache lock
    """
    
    self.__cacheLock.release( self.__class__.__name__)
  
  #.............................................................................
  # Cache getters

  def get( self, cacheKeys ):
    """
    Gets values for cacheKeys given, if all are found ( present on the cache and
    valid ), returns S_OK with the results. If any is not neither present not
    valid, returns S_ERROR. 
    
    :Parameters:
      **cacheKeys** - `list`
        list of keys to be extracted from the cache
        
    :return: S_OK | S_ERROR
    """

    result = {}

    for cacheKey in cacheKeys:

      cacheRow = self.__cache.get( cacheKey, validSeconds = self.__validSeconds )
      if not cacheRow:
        self.log.error( str( cacheKey ) )
        return S_ERROR( 'Cannot get %s' % str( cacheKey ) )
      result.update( { cacheKey : cacheRow } )
      
    return S_OK( result )

  #.............................................................................
  # Cache refreshers

  def refreshCache( self ):
    """     
    Purges the cache and gets fresh data from the update function.
#.........这里部分代码省略.........