Python ProcessPool.processResults方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import processResults [as 别名]
class PoolComputingElement(ComputingElement):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__(self, ceUniqueID):
    """ Standard constructor.
    """
    ComputingElement.__init__(self, ceUniqueID)
    self.ceType = "Pool"
    self.log = gLogger.getSubLogger('Pool')
    self.submittedJobs = 0
    self.processors = 1
    self.pPool = None
    self.taskID = 0
    self.processorsPerTask = {}
    self.userNumberPerTask = {}
    self.useSudo = False

  #############################################################################
  def _addCEConfigDefaults(self):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults(self)

  def _reset(self):
    """ Update internal variables after some extra parameters are added

    :return: None
    """

    self.processors = int(self.ceParameters.get('NumberOfProcessors', self.processors))
    self.ceParameters['MaxTotalJobs'] = self.processors
    self.useSudo = self.ceParameters.get('SudoExecution', False)

  def getProcessorsInUse(self):
    """ Get the number of currently allocated processor cores

    :return: number of processor cores
    """
    processorsInUse = 0
    for task in self.processorsPerTask:
      processorsInUse += self.processorsPerTask[task]
    return processorsInUse

  #############################################################################
  def submitJob(self, executableFile, proxy, **kwargs):
    """ Method to submit job.

    :param str executableFile: location of the executable file
    :param str proxy: payload proxy

    :return: S_OK/S_ERROR of the result of the job submission
    """

    if self.pPool is None:
      self.pPool = ProcessPool(minSize=self.processors,
                               maxSize=self.processors,
                               poolCallback=self.finalizeJob)

    self.pPool.processResults()

    processorsInUse = self.getProcessorsInUse()
    if kwargs.get('wholeNode'):
      if processorsInUse > 0:
        return S_ERROR('Can not take WholeNode job')  # , %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedProcessors = self.processors
    elif "numberOfProcessors" in kwargs:
      requestedProcessors = int(kwargs['numberOfProcessors'])
      if requestedProcessors > 0:
        if (processorsInUse + requestedProcessors) > self.processors:
          return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                           self.processors - processorsInUse))
    else:
      requestedProcessors = 1
    if self.processors - processorsInUse < requestedProcessors:
      return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                       self.processors - processorsInUse))

    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice('Pilot Proxy:', pilotProxy)

    kwargs = {'UseSudo': False}
    if self.useSudo:
      for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS):
        if nUser not in self.userNumberPerTask.values():
          break
      kwargs['NUser'] = nUser
      kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2)
      kwargs['UseSudo'] = True

    result = self.pPool.createAndQueueTask(executeJob,
                                           args=(executableFile, proxy, self.taskID),
                                           kwargs=kwargs,
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import processResults [as 别名]
class PoolComputingElement(ComputingElement):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__(self, ceUniqueID):
    """ Standard constructor.
    """
    ComputingElement.__init__(self, ceUniqueID)
    self.ceType = "Pool"
    self.log = gLogger.getSubLogger('Pool')
    self.submittedJobs = 0
    self.processors = 1
    self.pPool = None
    self.taskID = 0
    self.processorsPerTask = {}
    self.userNumberPerTask = {}
    self.useSudo = False

  #############################################################################
  def _addCEConfigDefaults(self):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults(self)

  def _reset(self):

    self.processors = int(self.ceParameters.get('NumberOfProcessors', self.processors))
    self.ceParameters['MaxTotalJobs'] = self.processors
    self.useSudo = self.ceParameters.get('SudoExecution', False)

  def getProcessorsInUse(self):
    """
    """
    processorsInUse = 0
    for task in self.processorsPerTask:
      processorsInUse += self.processorsPerTask[task]
    return processorsInUse

  #############################################################################
  def submitJob(self, executableFile, proxy, **kwargs):
    """ Method to submit job.
    """

    if self.pPool is None:
      self.pPool = ProcessPool(minSize=self.processors,
                               maxSize=self.processors,
                               poolCallback=self.finalizeJob)

    self.pPool.processResults()

    processorsInUse = self.getProcessorsInUse()
    if kwargs.get('wholeNode'):
      if processorsInUse > 0:
        return S_ERROR('Can not take WholeNode job')  # , %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedProcessors = self.processors
    elif "numberOfProcessors" in kwargs:
      requestedProcessors = int(kwargs['numberOfProcessors'])
      if requestedProcessors > 0:
        if (processorsInUse + requestedProcessors) > self.processors:
          return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                           self.processors - processorsInUse))
    else:
      requestedProcessors = 1
    if self.processors - processorsInUse < requestedProcessors:
      return S_ERROR('Not enough slots: requested %d, available %d' % (requestedProcessors,
                                                                       self.processors - processorsInUse))

    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice('Pilot Proxy:', pilotProxy)

    kwargs = {'UseSudo': False}
    if self.useSudo:
      for nUser in range(MAX_NUMBER_OF_SUDO_UNIX_USERS):
        if nUser not in self.userNumberPerTask.values():
          break
      kwargs['NUser'] = nUser
      kwargs['PayloadUser'] = os.environ['USER'] + 'p%s' % str(nUser).zfill(2)
      kwargs['UseSudo'] = True

    result = self.pPool.createAndQueueTask(executeJob,
                                           args=(executableFile, proxy, self.taskID),
                                           kwargs=kwargs,
                                           taskID=self.taskID,
                                           usePoolCallbacks=True)
    self.processorsPerTask[self.taskID] = requestedProcessors
    self.taskID += 1

    self.pPool.processResults()

    return result

  def finalizeJob(self, taskID, result):
    """ Finalize the job
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import processResults [as 别名]
#             'lfc-lhcb.grid.sara.nl',
#             'lfclhcb.pic.es',
#             'lhcb-lfc.gridpp.rl.ac.uk']
lfcHosts = ['prod-lfc-lhcb-ro.cern.ch']


# path = "/lhcb/LHCb"
path = '/lhcb/user/c/chaen'
print "Queueing task for directory", path, lfcHosts[0]


writerProc = Process( target = writer, args = ( 'lfc_dfc.out', writerQueue, stopFlag ) )

writerProc.start()


result = pPool.createAndQueueTask( processDir, [path , writerQueue, False, lfcHosts[0]], callback = finalizeDirectory )
if not result['OK']:
  print "Failed queueing", path

for i in range(20):
  pPool.processResults()
  time.sleep(1)

pPool.processAllResults( timeout = 300 )

stopFlag.value = 1
writerQueue.put( "Exit" )
writerProc.join()

# 需要导入模块: from DIRAC.Core.Utilities.ProcessPool import ProcessPool [as 别名]
# 或者: from DIRAC.Core.Utilities.ProcessPool.ProcessPool import processResults [as 别名]
class PoolComputingElement( ComputingElement ):

  mandatoryParameters = MandatoryParameters

  #############################################################################
  def __init__( self, ceUniqueID, cores = 0 ):
    """ Standard constructor.
    """
    ComputingElement.__init__( self, ceUniqueID )
    self.ceType = "Pool"
    self.submittedJobs = 0
    if cores > 0:
      self.cores = cores
    else:  
      self.cores = getNumberOfCores()
    self.pPool = ProcessPool( self.cores, self.cores, poolCallback = self.finalizeJob )
    self.taskID = 0
    self.coresPerTask = {}

  #############################################################################
  def _addCEConfigDefaults( self ):
    """Method to make sure all necessary Configuration Parameters are defined
    """
    # First assure that any global parameters are loaded
    ComputingElement._addCEConfigDefaults( self )
    
  def getCoresInUse( self ):
    """ 
    """ 
    coresInUse = 0
    for _task, cores in self.coresPerTask.items():
      coresInUse += cores 
    return coresInUse  

  #############################################################################
  def submitJob( self, executableFile, proxy, **kwargs ):
    """ Method to submit job, should be overridden in sub-class.
    """
    
    self.pPool.processResults()
    
    coresInUse = self.getCoresInUse()
    if "WholeNode" in kwargs and kwargs['WholeNode']:
      if coresInUse > 0:
        return S_ERROR('Can not take WholeNode job, %d/%d slots used' % (self.slotsInUse,self.slots) )
      else:
        requestedCores = self.cores
    elif "NumberOfCores" in kwargs:           
      requestedCores = int( kwargs['NumberOfCores'] )
      if requestedCores > 0:
        if (coresInUse + requestedCores) > self.cores:
          return S_ERROR( 'Not enough slots: requested %d, available %d' % ( requestedCores, self.cores-coresInUse) )
    else:
      requestedCores = 1   
    if self.cores - coresInUse < requestedCores:
      return S_ERROR( 'Not enough slots: requested %d, available %d' % ( requestedCores, self.cores-coresInUse) )
    
    ret = getProxyInfo()
    if not ret['OK']:
      pilotProxy = None
    else:
      pilotProxy = ret['Value']['path']
    self.log.notice( 'Pilot Proxy:', pilotProxy )

    result = self.pPool.createAndQueueTask( executeJob, 
                                            [executableFile,proxy,self.taskID],None,
                                            self.taskID,
                                            usePoolCallbacks = True )
    self.taskID += 1
    self.coresPerTask[self.taskID] = requestedCores
    
    self.pPool.processResults()
    
    return result
  
  def finalizeJob( self, taskID, result ):
    """ Finalize the job
    """
    del self.coresPerTask[taskID]

  #############################################################################
  def getCEStatus( self ):
    """ Method to return information on running and pending jobs.
    """
    self.pPool.processResults()
    result = S_OK()
    result['SubmittedJobs'] = 0
    nJobs = 0
    for _j, value in self.coresPerTask.items():
      if value > 0: 
        nJobs += 1
    result['RunningJobs'] = nJobs 
    result['WaitingJobs'] = 0
    coresInUse = self.getCoresInUse()
    result['UsedCores'] = coresInUse
    result['AvailableCores'] = self.cores - coresInUse
    return result

  #############################################################################
  def monitorProxy( self, pilotProxy, payloadProxy ):
#.........这里部分代码省略.........