Python ProcessPool.daemonize方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]
class TaskCallbacksTests(unittest.TestCase):
  """
  .. class:: TaskCallbacksTests
  test case for ProcessPool
  """

  def setUp( self ):
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 4, 8, 8 ) 
    self.processPool.daemonize()

  def testCallableClass( self ):
    """ CallableClass and task callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      callback = ResultCallback,
                                                      exceptionCallback = ExceptionCallback,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

  def testCallableFunc( self ):
    """ CallableFunc and task callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      callback = ResultCallback,
                                                      exceptionCallback = ExceptionCallback,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]
argvs = sys.argv
if len(argvs) != 2 :
   print 'Usage: LFC_to_DFC.py [dirlist_file]'
   print '[dirlist_file] should contain the directory list.'
   quit()

dirlist_file=argvs[1]
if ( not os.path.exists(dirlist_file) ) :
  print dirlist_file+" does not exist"
  quit()

execfile(dirlist_file)

pPool = ProcessPool(10,50,50)
pPool.daemonize()

# dirlist = ['prod/ilc/mc-dbd/generated','prod/ilc/mc-dbd/ild']
# dirlist= ['prod/ilc/mc-dbd/generated/500-TDR_ws/higgs']
# dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/higgs','prod/ilc/mc-dbd/generated/350-TDR_ws/higgs']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/1f',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/3f',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_lowpt',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_minijet']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/aa_2f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/3f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/1f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/aa_minijet']

lfcHosts = ['grid-lfc.desy.de']

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]

#.........这里部分代码省略.........
      opHandler = item[1][0]
      self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1],
                                                                   self.timeOuts[opHandler]['PerOperation'],
                                                                   self.timeOuts[opHandler]['PerFile'] ) )

    # # common monitor activity
    gMonitor.registerActivity( "Iteration", "Agent Loops",
                               "RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Processed", "Request Processed",
                               "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Done", "Request Completed",
                               "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
    # # create request dict
    self.__requestCache = dict()

    self.FTSMode = self.am_getOption( "FTSMode", False )



  def processPool( self ):
    """ facade for ProcessPool """
    if not self.__processPool:
      minProcess = max( 1, self.__minProcess )
      maxProcess = max( self.__minProcess, self.__maxProcess )
      queueSize = abs( self.__queueSize )
      self.log.info( "ProcessPool: minProcess = %d maxProcess = %d queueSize = %d" % ( minProcess,
                                                                                       maxProcess,
                                                                                       queueSize ) )
      self.__processPool = ProcessPool( minProcess,
                                        maxProcess,
                                        queueSize,
                                        poolCallback = self.resultCallback,
                                        poolExceptionCallback = self.exceptionCallback )
      self.__processPool.daemonize()
    return self.__processPool

  def requestClient( self ):
    """ RequestClient getter """
    if not self.__requestClient:
      self.__requestClient = ReqClient()
    return self.__requestClient

  def cacheRequest( self, request ):
    """ put request into requestCache

    :param Request request: Request instance
    """
    count = 5
    # Wait a bit as there may be a race condition between RequestTask putting back the request and the callback clearing the cache
    while request.RequestID in self.__requestCache:
      count -= 1
      if not count:
        self.requestClient().putRequest( request, useFailoverProxy = False, retryMainService = 2 )
        return S_ERROR( "Duplicate request, ignore: %s" % request.RequestID )
      time.sleep( 1 )
    self.__requestCache[ request.RequestID ] = request
    return S_OK()

  def putRequest( self, requestID, taskResult = None ):
    """ put back :requestID: to RequestClient

    :param str requestID: request's id
    """
    if requestID in self.__requestCache:
      request = self.__requestCache.pop( requestID )
      if taskResult and taskResult['OK']:

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]
class TaskTimeOutTests( unittest.TestCase ):
  """
  .. class:: TaskTimeOutTests

  test case for ProcessPool
  """

  def setUp( self ):
    """c'tor

    :param self: self reference
    """
    from DIRAC.Core.Base import Script
    Script.parseCommandLine()
    from DIRAC.FrameworkSystem.Client.Logger import gLogger
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 2,
                                    4, 
                                    8,
                                    poolCallback = self.poolCallback, 
                                    poolExceptionCallback = self.poolExceptionCallback )
    self.processPool.daemonize()
    
  def poolCallback( self, taskID, taskResult ):
    self.log.always( "callback result for %s is %s" % ( taskID, taskResult )  ) 

  def poolExceptionCallback( self, taskID, taskException ): 
    self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) )


  def testCallableClass( self ):
    """ CallableClass and task time out test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint( 0, 5 ) * 10
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ), 
                                                      timeOut = 15,
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
          i += 1
        else:
          continue
      if i == 16:
        break
    self.processPool.finalize( 2 )
    
  def testCallableFunc( self ):
    """ CallableFunc and task timeout test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5) * 5
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      timeOut = 15,
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
          i += 1
        else:
          continue
      if i == 16:
        break
    self.processPool.finalize( 2 )

  def testLockedClass( self ):
    """ LockedCallableClass and task time out test """

    for loop in range(2):
      self.log.always( "loop %s" % loop )
      i = 0
      while i < 16:
        if self.processPool.getFreeSlots() > 0:
          timeWait = random.randint(0, 5) * 5
          raiseException = False
          if timeWait == 5:
            raiseException = True
          klass = CallableClass
          if timeWait >= 20:
            klass = LockedCallableClass
          result = self.processPool.createAndQueueTask( klass,
                                                        taskID = i,
                                                        args = ( i, timeWait, raiseException ), 
                                                        timeOut = 15,
                                                        usePoolCallbacks = True,
                                                        blocking = True )    
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]
class ProcessPoolCallbacksTests( unittest.TestCase ):
  """
  .. class:: ProcessPoolCallbacksTests
  test case for ProcessPool
  """

  def setUp( self ):
    """c'tor

    :param self: self reference
    """
    from DIRAC.Core.Base import Script
    Script.parseCommandLine()
    from DIRAC.FrameworkSystem.Client.Logger import gLogger
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 4, 8, 8,
                                    poolCallback = self.poolCallback, 
                                    poolExceptionCallback = self.poolExceptionCallback )
    self.processPool.daemonize()

  def poolCallback( self, taskID, taskResult ):
    self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) ) 

  def poolExceptionCallback( self, taskID, taskException ):
    self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) )

  def testCallableClass( self ):
    """ CallableClass and pool callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )


  def testCallableFunc( self ):
    """ CallableFunc and pool callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableFunc enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]

#.........这里部分代码省略.........
    """
    if requestName in self.__requestHolder:
      requestString = self.__requestHolder[requestName]
      reset = self.requestClient().updateRequest( requestName, requestString )
      if not reset["OK"]:
        self.log.error("resetRequest: unable to reset request %s: %s" % ( requestName, reset["Message"] ) )
      self.log.debug("resetRequest: request %s has been put back with its initial state" % requestName )
    else:
      self.log.error("resetRequest: unable to reset request %s: request not found in requestHolder" % requestName )

  def resetAllRequests( self ):
    """ put back all requests without callback called into requestClient 

    :param self: self reference
    """
    self.log.info("resetAllRequests: will put %s back requests" % len(self.__requestHolder) )
    for requestName, requestString in self.__requestHolder.items():
      reset = self.requestClient().updateRequest( requestName, requestString )
      if not reset["OK"]:
        self.log.error("resetAllRequests: unable to reset request %s: %s" % ( requestName, reset["Message"] ) )
        continue
      self.log.debug("resetAllRequests: request %s has been put back with its initial state" % requestName )

  def configPath( self ):
    """ config path getter

    :param self: self reference
    """
    return self.__configPath

  def requestsPerCycle( self ):
    """ get number of request to be processed in one cycle
    
    :param self: self reference
    """
    return self.__requestsPerCycle

  def requestClient( self ):
    """ RequestClient getter

    :param self: self reference
    """
    if not self.__requestClient:
      self.__requestClient = RequestClient()
    return self.__requestClient

  def processPool( self ):
    """ 'Live long and prosper, my dear ProcessPool'
                                        - Mr. Spock    
    :param self: self reference
    :return: brand new shiny ProcessPool instance on first call, the same instance
             on subsequent calls
    """
    if not self.__processPool:
      minProcess = max( 1, self.__minProcess ) 
      maxProcess = max( self.__minProcess, self.__maxProcess )
      queueSize = abs(self.__queueSize) 
      self.log.info( "ProcessPool: minProcess = %d maxProcess = %d queueSize = %d" % ( minProcess, 
                                                                                       maxProcess, 
                                                                                       queueSize ) )
      self.log.info( "ProcessPool: tasks will use callbacks attached to ProcessPool" )
      self.__processPool = ProcessPool( minProcess, 
                                        maxProcess, 
                                        queueSize, 
                                        poolCallback = self.resultCallback,
                                        poolExceptionCallback = self.exceptionCallback )
      self.__processPool.daemonize()
      self.log.info( "ProcessPool: daemonized and ready")
    return self.__processPool

  def hasProcessPool( self ):
    """ check if ProcessPool exist to speed up finalization 

    :param self: self reference
    """
    return bool( self.__processPool )

  def resultCallback( self, taskID, taskResult ):
    """ definition of request callback function
    
    :param self: self reference
    """
    self.log.info("%s result callback" %  taskID ) 

    if not taskResult["OK"]:
      self.log.error( "%s result callback: %s" % ( taskID, taskResult["Message"] ) )
      if taskResult["Message"] == "Timed out":
        self.resetRequest( taskID )
      self.deleteRequest( taskID )
      return
    
    self.deleteRequest( taskID )
    taskResult = taskResult["Value"]
    ## add monitoring info
    monitor = taskResult["monitor"] if "monitor" in taskResult else {}
    for mark, value in monitor.items():
      try:
        gMonitor.addMark( mark, value )
      except Exception, error:
        self.log.exception( str(error) )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]

#.........这里部分代码省略.........

    self.log.info( "Operation handlers:" )
    for itemTuple in enumerate ( self.operationHandlers ):
      self.log.info( "[%s] %s" % itemTuple )

    # # handlers dict
    self.handlersDict = dict()
    # # common monitor activity
    gMonitor.registerActivity( "Iteration", "Agent Loops",
                               "RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Processed", "Request Processed",
                               "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Done", "Request Completed",
                               "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
    # # create request dict
    self.__requestCache = dict()

    self.FTSMode = self.am_getOption( "FTSMode", False )

  def processPool( self ):
    """ facade for ProcessPool """
    if not self.__processPool:
      minProcess = max( 1, self.__minProcess )
      maxProcess = max( self.__minProcess, self.__maxProcess )
      queueSize = abs( self.__queueSize )
      self.log.info( "ProcessPool: minProcess = %d maxProcess = %d queueSize = %d" % ( minProcess,
                                                                                       maxProcess,
                                                                                       queueSize ) )
      self.__processPool = ProcessPool( minProcess,
                                        maxProcess,
                                        queueSize,
                                        poolCallback = self.resultCallback,
                                        poolExceptionCallback = self.exceptionCallback )
      self.__processPool.daemonize()
    return self.__processPool

  def requestClient( self ):
    """ RequestClient getter """
    if not self.__requestClient:
      self.__requestClient = ReqClient()
    return self.__requestClient

  def cleanCache( self, requestName = None ):
    """ delete request from requestCache

    :param str requestName: Request.RequestName
    """
    if requestName in self.__requestCache:
      del self.__requestCache[requestName]
    return S_OK()

  def cacheRequest( self, request ):
    """ put request into requestCache

    :param Request request: Request instance
    """
    self.__requestCache.setdefault( request.RequestName, request )
    return S_OK()

  def resetRequest( self, requestName ):
    """ put back :requestName: to RequestClient

    :param str requestName: request's name
    """
    if requestName in self.__requestCache:
      reset = self.requestClient().updateRequest( self.__requestCache[requestName] )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import daemonize [as 别名]

#.........这里部分代码省略.........
    for item in enumerate(self.handlersDict.items()):
      opHandler = item[1][0]
      self.log.info("[%s] %s: %s (timeout: %d s + %d s per file)" % (item[0], item[1][0], item[1][1],
                                                                     self.timeOuts[opHandler]['PerOperation'],
                                                                     self.timeOuts[opHandler]['PerFile']))

    # # common monitor activity
    gMonitor.registerActivity("Iteration", "Agent Loops",
                              "RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM)
    gMonitor.registerActivity("Processed", "Request Processed",
                              "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
    gMonitor.registerActivity("Done", "Request Completed",
                              "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
    # # create request dict
    self.__requestCache = dict()

    # ?? Probably should be removed
    self.FTSMode = self.am_getOption("FTSMode", False)

  def processPool(self):
    """ facade for ProcessPool """
    if not self.__processPool:
      minProcess = max(1, self.__minProcess)
      maxProcess = max(self.__minProcess, self.__maxProcess)
      queueSize = abs(self.__queueSize)
      self.log.info("REA ProcessPool configuration", "minProcess = %d maxProcess = %d queueSize = %d" % (minProcess,
                                                                                                         maxProcess,
                                                                                                         queueSize))
      self.__processPool = ProcessPool(minProcess,
                                       maxProcess,
                                       queueSize,
                                       poolCallback=self.resultCallback,
                                       poolExceptionCallback=self.exceptionCallback)
      self.__processPool.daemonize()
    return self.__processPool

  def requestClient(self):
    """ RequestClient getter """
    if not self.__requestClient:
      self.__requestClient = ReqClient()
    return self.__requestClient

  def cacheRequest(self, request):
    """ put request into requestCache

    :param ~Request.Request request: Request instance
    """
    maxProcess = max(self.__minProcess, self.__maxProcess)
    if len(self.__requestCache) > maxProcess + 50:
      # For the time being we just print a warning... If the ProcessPool is working well, this is not needed
      # We don't know how much is acceptable as it depends on many factors
      self.log.warn("Too many requests in cache", ': %d' % len(self.__requestCache))
#      return S_ERROR( "Too many requests in cache" )
    if request.RequestID in self.__requestCache:
      # We don't call  putRequest as we have got back the request that is still being executed. Better keep it
      # The main reason for this is that it lasted longer than the kick time of CleanReqAgent
      self.log.warn("Duplicate request, keep it but don't execute",
                    ': %d/%s' % (request.RequestID, request.RequestName))
      return S_ERROR(errno.EALREADY, 'Request already in cache')
    self.__requestCache[request.RequestID] = request
    return S_OK()

  def putRequest(self, requestID, taskResult=None):
    """ put back :requestID: to RequestClient

    :param str requestID: request's id