Python ProcessPool.createAndQueueTask方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class TaskCallbacksTests(unittest.TestCase):
  """
  .. class:: TaskCallbacksTests
  test case for ProcessPool
  """

  def setUp( self ):
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 4, 8, 8 ) 
    self.processPool.daemonize()

  def testCallableClass( self ):
    """ CallableClass and task callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      callback = ResultCallback,
                                                      exceptionCallback = ExceptionCallback,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

  def testCallableFunc( self ):
    """ CallableFunc and task callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      callback = ResultCallback,
                                                      exceptionCallback = ExceptionCallback,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
def runTest():

  global nClients, nQueries, testType, resultTest, testDir, lfnListFile

  resultTest = []

  pp = ProcessPool(nClients)

  testFunction = eval(testType)

  for c in xrange(nClients):
    pp.createAndQueueTask(testFunction, [nQueries],
                          callback=finalize,
                          exceptionCallback=doException)

  pp.processAllResults(3600)
  pp.finalize(0)

  timeResult = []
  for testTime, success, failure in resultTest:
    # print testTime,success,failure
    timeResult += testTime

  averageTime, errorTime = doStats(timeResult)
  rateResult = [nClients / t for t in timeResult]
  averageRate, errorRate = doStats(rateResult)

  if testDir:
    print "\nTest results for clients %d, %s" % (nClients, testDir)
  else:
    print "\nTest results for clients %d, %s" % (nClients, lfnListFile)

  print "Query time: %.2f +/- %.2f" % (averageTime, errorTime)
  print "Query rate: %.2f +/- %.2f" % (averageRate, errorRate)

  return((averageTime, errorTime), (averageRate, errorRate))

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class PoolComputingElement(ComputingElement):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__(self, ceUniqueID):
    """ Standard constructor.
    """
    ComputingElement.__init__(self, ceUniqueID)
    self.ceType = "Pool"
    self.log = gLogger.getSubLogger('Pool')
    self.submittedJobs = 0
    self.processors = 1
    self.pPool = None
    self.taskID = 0
    self.processorsPerTask = {}
    self.userNumberPerTask = {}
    self.useSudo = False

  #############################################################################
  def _addCEConfigDefaults(self):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults(self)

  def _reset(self):
    """ Update internal variables after some extra parameters are added

    :return: None
    """

    self.processors = int(self.ceParameters.get('NumberOfProcessors', self.processors))
    self.ceParameters['MaxTotalJobs'] = self.processors
    self.useSudo = self.ceParameters.get('SudoExecution', False)

  def getProcessorsInUse(self):
    """ Get the number of currently allocated processor cores

    :return: number of processor cores
    """
    processorsInUse = 0
    for task in self.processorsPerTask:
      processorsInUse += self.processorsPerTask[task]
    return processorsInUse

  #############################################################################
  def submitJob(self, executableFile, proxy, **kwargs):
    """ Method to submit job.

    :param str executableFile: location of the executable file
    :param str proxy: payload proxy

    :return: S_OK/S_ERROR of the result of the job submission
    """

    if self.pPool is None:
      self.pPool = ProcessPool(minSize=self.processors,
                               maxSize=self.processors,
                               poolCallback=self.finalizeJob)

    self.pPool.processResults()

    processorsInUse = self.getProcessorsInUse()
    if kwargs.get('wholeNode'):
      if processorsInUse > 0:
        return S_ERROR('Can not take WholeNode job')  # , %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedProcessors = self.processors
    elif "numberOfProcessors" in kwargs:
      requestedProcessors = int(kwargs['numberOfProcessors'])
      if requestedProcessors > 0:
        if (processorsInUse + requestedProcessors) > self.processors:
          return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                           self.processors - processorsInUse))
    else:
      requestedProcessors = 1
    if self.processors - processorsInUse < requestedProcessors:
      return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                       self.processors - processorsInUse))

    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice('Pilot Proxy:', pilotProxy)

    kwargs = {'UseSudo': False}
    if self.useSudo:
      for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS):
        if nUser not in self.userNumberPerTask.values():
          break
      kwargs['NUser'] = nUser
      kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2)
      kwargs['UseSudo'] = True

    result = self.pPool.createAndQueueTask(executeJob,
                                           args=(executableFile, proxy, self.taskID),
                                           kwargs=kwargs,
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
pPool = ProcessPool(10,50,50)
pPool.daemonize()

# dirlist = ['prod/ilc/mc-dbd/generated','prod/ilc/mc-dbd/ild']
# dirlist= ['prod/ilc/mc-dbd/generated/500-TDR_ws/higgs']
# dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/higgs','prod/ilc/mc-dbd/generated/350-TDR_ws/higgs']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/1f',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/3f',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_lowpt',
#          'prod/ilc/mc-dbd/generated/250-TDR_ws/aa_minijet']
#dirlist= ['prod/ilc/mc-dbd/generated/250-TDR_ws/aa_2f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/3f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/1f',
#          'prod/ilc/mc-dbd/generated/350-TDR_ws/aa_minijet']

lfcHosts = ['grid-lfc.desy.de']

for dir in dirlist:
  path = "/ilc/%s" % (dir)
  print "Queueing user", dir, pPool.getFreeSlots(),pPool.getNumWorkingProcesses(),pPool.hasPendingTasks(),pPool.getNumIdleProcesses(), lfcHosts[0]
  result = pPool.createAndQueueTask( processDir,[path,True,lfcHosts[0]],callback=finalizeDirectory ) 
  if not result['OK']:
    print "Failed queueing", path

pPool.processAllResults()

print "LFC_to_DFC completed."

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class TaskTimeOutTests( unittest.TestCase ):
  """
  .. class:: TaskTimeOutTests

  test case for ProcessPool
  """

  def setUp( self ):
    """c'tor

    :param self: self reference
    """
    from DIRAC.Core.Base import Script
    Script.parseCommandLine()
    from DIRAC.FrameworkSystem.Client.Logger import gLogger
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 2,
                                    4, 
                                    8,
                                    poolCallback = self.poolCallback, 
                                    poolExceptionCallback = self.poolExceptionCallback )
    self.processPool.daemonize()
    
  def poolCallback( self, taskID, taskResult ):
    self.log.always( "callback result for %s is %s" % ( taskID, taskResult )  ) 

  def poolExceptionCallback( self, taskID, taskException ): 
    self.log.always( "callback exception for %s is %s" % ( taskID, taskException ) )


  def testCallableClass( self ):
    """ CallableClass and task time out test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint( 0, 5 ) * 10
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ), 
                                                      timeOut = 15,
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
          i += 1
        else:
          continue
      if i == 16:
        break
    self.processPool.finalize( 2 )
    
  def testCallableFunc( self ):
    """ CallableFunc and task timeout test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5) * 5
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      timeOut = 15,
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableFunc enqueued to task %s timeWait=%s exception=%s" % ( i, timeWait, raiseException ) )
          i += 1
        else:
          continue
      if i == 16:
        break
    self.processPool.finalize( 2 )

  def testLockedClass( self ):
    """ LockedCallableClass and task time out test """

    for loop in range(2):
      self.log.always( "loop %s" % loop )
      i = 0
      while i < 16:
        if self.processPool.getFreeSlots() > 0:
          timeWait = random.randint(0, 5) * 5
          raiseException = False
          if timeWait == 5:
            raiseException = True
          klass = CallableClass
          if timeWait >= 20:
            klass = LockedCallableClass
          result = self.processPool.createAndQueueTask( klass,
                                                        taskID = i,
                                                        args = ( i, timeWait, raiseException ), 
                                                        timeOut = 15,
                                                        usePoolCallbacks = True,
                                                        blocking = True )    
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class ProcessPoolCallbacksTests( unittest.TestCase ):
  """
  .. class:: ProcessPoolCallbacksTests
  test case for ProcessPool
  """

  def setUp( self ):
    """c'tor

    :param self: self reference
    """
    from DIRAC.Core.Base import Script
    Script.parseCommandLine()
    from DIRAC.FrameworkSystem.Client.Logger import gLogger
    gLogger.showHeaders( True )
    self.log = gLogger.getSubLogger( self.__class__.__name__ )
    self.processPool = ProcessPool( 4, 8, 8,
                                    poolCallback = self.poolCallback, 
                                    poolExceptionCallback = self.poolExceptionCallback )
    self.processPool.daemonize()

  def poolCallback( self, taskID, taskResult ):
    self.log.always( "callback for %s result is %s" % ( taskID, taskResult ) ) 

  def poolExceptionCallback( self, taskID, taskException ):
    self.log.always( "callback for %s exception is %s" % ( taskID, taskException ) )

  def testCallableClass( self ):
    """ CallableClass and pool callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableClass,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableClass enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )


  def testCallableFunc( self ):
    """ CallableFunc and pool callbacks test """
    i = 0
    while True:
      if self.processPool.getFreeSlots() > 0:
        timeWait = random.randint(0, 5)
        raiseException = False
        if not timeWait:
          raiseException = True 
        result = self.processPool.createAndQueueTask( CallableFunc,
                                                      taskID = i,
                                                      args = ( i, timeWait, raiseException ),  
                                                      usePoolCallbacks = True,
                                                      blocking = True )    
        if result["OK"]:
          self.log.always("CallableFunc enqueued to task %s" % i )
          i += 1          
        else:
          continue
      if i == 10:
        break
    self.processPool.finalize( 2 )

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class PoolComputingElement(ComputingElement):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__(self, ceUniqueID):
    """ Standard constructor.
    """
    ComputingElement.__init__(self, ceUniqueID)
    self.ceType = "Pool"
    self.log = gLogger.getSubLogger('Pool')
    self.submittedJobs = 0
    self.processors = 1
    self.pPool = None
    self.taskID = 0
    self.processorsPerTask = {}
    self.userNumberPerTask = {}
    self.useSudo = False

  #############################################################################
  def _addCEConfigDefaults(self):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults(self)

  def _reset(self):

    self.processors = int(self.ceParameters.get('NumberOfProcessors', self.processors))
    self.ceParameters['MaxTotalJobs'] = self.processors
    self.useSudo = self.ceParameters.get('SudoExecution', False)

  def getProcessorsInUse(self):
    """
    """
    processorsInUse = 0
    for task in self.processorsPerTask:
      processorsInUse += self.processorsPerTask[task]
    return processorsInUse

  #############################################################################
  def submitJob(self, executableFile, proxy, **kwargs):
    """ Method to submit job.
    """

    if self.pPool is None:
      self.pPool = ProcessPool(minSize=self.processors,
                               maxSize=self.processors,
                               poolCallback=self.finalizeJob)

    self.pPool.processResults()

    processorsInUse = self.getProcessorsInUse()
    if kwargs.get('wholeNode'):
      if processorsInUse > 0:
        return S_ERROR('Can not take WholeNode job')  # , %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedProcessors = self.processors
    elif "numberOfProcessors" in kwargs:
      requestedProcessors = int(kwargs['numberOfProcessors'])
      if requestedProcessors > 0:
        if (processorsInUse + requestedProcessors) > self.processors:
          return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                           self.processors - processorsInUse))
    else:
      requestedProcessors = 1
    if self.processors - processorsInUse < requestedProcessors:
      return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                       self.processors - processorsInUse))

    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice('Pilot Proxy:', pilotProxy)

    kwargs = {'UseSudo': False}
    if self.useSudo:
      for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS):
        if nUser not in self.userNumberPerTask.values():
          break
      kwargs['NUser'] = nUser
      kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2)
      kwargs['UseSudo'] = True

    result = self.pPool.createAndQueueTask(executeJob,
                                           args=(executableFile, proxy, self.taskID),
                                           kwargs=kwargs,
                                           taskID=self.taskID,
                                           usePoolCallbacks=True)
    self.processorsPerTask[self.taskID] = requestedProcessors
    self.taskID += 1

    self.pPool.processResults()

    return result

  def finalizeJob(self, taskID, result):
    """ Finalize the job
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import createAndQueueTask [as 别名]
class PoolComputingElement( ComputingElement ):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__( self, ceUniqueID, cores = 0 ):
    """ Standard constructor.
    """
    ComputingElement.__init__( self, ceUniqueID )
    self.ceType = "Pool"
    self.submittedJobs = 0
    if cores > 0:
      self.cores = cores
    else:  
      self.cores = getNumberOfCores()
    self.pPool = ProcessPool( self.cores, self.cores, poolCallback = self.finalizeJob )
    self.taskID = 0
    self.coresPerTask = {}

  #############################################################################
  def _addCEConfigDefaults( self ):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults( self )
    
  def getCoresInUse( self ):
    """ 
    """ 
    coresInUse = 0
    for _task, cores in self.coresPerTask.items():
      coresInUse += cores 
    return coresInUse  

  #############################################################################
  def submitJob( self, executableFile, proxy, **kwargs ):
    """ Method to submit job, should be overridden in sub-class.
    """
    
    self.pPool.processResults()
    
    coresInUse = self.getCoresInUse()
    if "WholeNode" in kwargs and kwargs['WholeNode']:
      if coresInUse > 0:
        return S_ERROR('Can not take WholeNode job, %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedCores = self.cores
    elif "NumberOfCores" in kwargs:           
      requestedCores = int( kwargs['NumberOfCores'] )
      if requestedCores > 0:
        if (coresInUse + requestedCores) > self.cores:
          return S_ERROR( 'Not enough slots: requested %d, available %d' % ( requestedCores, self.cores-coresInUse) )
    else:
      requestedCores = 1   
    if self.cores - coresInUse < requestedCores:
      return S_ERROR( 'Not enough slots: requested %d, available %d' % ( requestedCores, self.cores-coresInUse) )
    
    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice( 'Pilot Proxy:', pilotProxy )

    result = self.pPool.createAndQueueTask( executeJob, 
                                            [executableFile,proxy,self.taskID],None,
                                            self.taskID,
                                            usePoolCallbacks = True )
    self.taskID += 1
    self.coresPerTask[self.taskID] = requestedCores
    
    self.pPool.processResults()
    
    return result
  
  def finalizeJob( self, taskID, result ):
    """ Finalize the job
    """
    del self.coresPerTask[taskID]

  #############################################################################
  def getCEStatus( self ):
    """ Method to return information on running and pending jobs.
    """
    self.pPool.processResults()
    result = S_OK()
    result['SubmittedJobs'] = 0
    nJobs = 0
    for _j, value in self.coresPerTask.items():
      if value > 0: 
        nJobs += 1
    result['RunningJobs'] = nJobs 
    result['WaitingJobs'] = 0
    coresInUse = self.getCoresInUse()
    result['UsedCores'] = coresInUse
    result['AvailableCores'] = self.cores - coresInUse
    return result

  #############################################################################
  def monitorProxy( self, pilotProxy, payloadProxy ):
#.........这里部分代码省略.........