Python ThreadPool.ThreadPool类代码示例

  def initialize( self ):
    """ Agent initialization.

        The extensions MUST provide in the initialize method the following data members:
        - TransformationClient objects (self.transClient),
        - set the shifterProxy if different from the default one set here ('ProductionManager')
        - list of transformation types to be looked (self.transType)
    """

    gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
                               gMonitor.OP_ACUM )

    self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin' )

    # Default clients
    self.transClient = TransformationClient()

    # Bulk submission flag
    self.bulkSubmissionFlag = self.am_getOption( 'BulkSubmission', False )

    # setting up the threading
    maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
    threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
    self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )

    for i in xrange( maxNumberOfThreads ):
      threadPool.generateJobAndQueueIt( self._execute, [i] )

    return S_OK()

class MyProxyRenewalAgent(AgentModule):

  def initialize(self):

    requiredLifeTime = self.am_getOption( "MinimumLifeTime", 3600 )
    renewedLifeTime = self.am_getOption( "RenewedLifeTime", 54000 )
    myProxyServer = gConfig.getValue( "/DIRAC/VOPolicy/MyProxyServer" , "myproxy.cern.ch" )
    self.proxyDB = ProxyDB( requireVoms = True,
                            useMyProxy = True
                          )

    gLogger.info( "Minimum Life time      : %s" % requiredLifeTime )
    gLogger.info( "Life time on renew     : %s" % renewedLifeTime )
    gLogger.info( "MyProxy server         : %s" % self.proxyDB.getMyProxyServer() )
    gLogger.info( "MyProxy max proxy time : %s" % self.proxyDB.getMyProxyMaxLifeTime() )

    self.__threadPool = ThreadPool( 1, 10 )
    return S_OK()

  def __renewProxyForCredentials( self, userDN, userGroup ):
    lifeTime = self.am_getOption( "RenewedLifeTime", 54000 )
    gLogger.info( "Renewing for %[email protected]%s %s secs" % ( userDN, userGroup, lifeTime ) )
    retVal = self.proxyDB.renewFromMyProxy( userDN,
                                            userGroup,
                                            lifeTime = lifeTime )
    if not retVal[ 'OK' ]:
      gLogger.error( "Failed to renew for %[email protected]%s : %s" %( userDN, userGroup, retVal[ 'Message' ] ) )
    else:
      gLogger.info( "Renewed proxy for %[email protected]%s" % ( userDN, userGroup ) )

  def __treatRenewalCallback( self, oTJ, exceptionList ):
    gLogger.exception( lException = exceptionList )

  def execute(self):
    """ The main agent execution method
    """
    self.proxyDB.purgeLogs()
    gLogger.info( "Purging expired requests" )
    retVal = self.proxyDB.purgeExpiredRequests()
    if retVal[ 'OK' ]:
      gLogger.info( " purged %s requests" % retVal[ 'Value' ] )
    gLogger.info( "Purging expired proxies" )
    retVal = self.proxyDB.purgeExpiredProxies()
    if retVal[ 'OK' ]:
      gLogger.info( " purged %s proxies" % retVal[ 'Value' ] )
    retVal = self.proxyDB.getCredentialsAboutToExpire( self.am_getOption( "MinimumLifeTime" , 3600 ) )
    if not retVal[ 'OK' ]:
      return retVal
    data = retVal[ 'Value' ]
    gLogger.info( "Renewing %s proxies..." % len( data ) )
    for record in data:
      userDN = record[0]
      userGroup = record[1]
      self.__threadPool.generateJobAndQueueIt( self.__renewProxyForCredentials,
                                               args = ( userDN, userGroup ),
                                               oExceptionCallback = self.__treatRenewalCallback )
    self.__threadPool.processAllResults()
    return S_OK()

class SystemAdministratorIntegrator:

  def __init__( self, **kwargs ):
    """ Constructor  
    """
    if 'hosts' in kwargs:
      self.__hosts = kwargs['hosts']
      del kwargs['hosts']
    else:  
      result = Registry.getHosts()
      if result['OK']:
        self.__hosts = result['Value']
      else:
        self.__hosts = []
      
    self.__kwargs = dict( kwargs )  
    self.__pool = ThreadPool( len( self.__hosts ) )  
    self.__resultDict = {}
      
  def __getattr__( self, name ):
    self.call = name
    return self.execute

  def __executeClient( self, host, method, *parms, **kwargs ):
    """ Execute RPC method on a given host 
    """        
    hostName = Registry.getHostOption( host, 'Host', host)
    client = SystemAdministratorClient( hostName, **self.__kwargs )
    result = getattr( client, method )( *parms, **kwargs )
    result['Host'] = host   
    return result
    
  def __processResult( self, id_, result ):
    """ Collect results in the final structure
    """
    host = result['Host']
    del result['Host']
    self.__resultDict[host] = result  
       
  def execute(self, *args, **kwargs ):
    """ Main execution method
    """
    self.__resultDict = {}
    for host in self.__hosts:
      self.__pool.generateJobAndQueueIt( self.__executeClient,
                                         args = [ host, self.call ] + list(args),
                                         kwargs = kwargs,
                                         oCallback = self.__processResult )
    
    self.__pool.processAllResults()
    return S_OK( self.__resultDict )    

  def __addPool( self, poolName ):
    # create a new thread Pool, by default it has 2 executing threads and 40 requests
    # in the Queue

    if not poolName:
      return None
    if poolName in self.pools:
      return None
    pool = ThreadPool( self.am_getOption( 'minThreadsInPool' ),
                       self.am_getOption( 'maxThreadsInPool' ),
                       self.am_getOption( 'totalThreadsInPool' ) )
    # Daemonize except "Default" pool
    if poolName != 'Default':
      pool.daemonize()
    self.pools[poolName] = pool
    return poolName

  def initialize( self ):
    #Build the URLs
    self._url = self._cfg.getURL()
    if not self._url:
      return S_ERROR( "Could not build service URL for %s" % self._name )
    gLogger.verbose( "Service URL is %s" % self._url )
    #Load handler
    result = self._loadHandlerInit()
    if not result[ 'OK' ]:
      return result
    self._handler = result[ 'Value' ]
    #Initialize lock manager
    self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
    self._initMonitoring()
    self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
                                   max( 0, self._cfg.getMaxThreads() ),
                                   self._cfg.getMaxWaitingPetitions() )
    self._threadPool.daemonize()
    self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
    #Create static dict
    self._serviceInfoDict = { 'serviceName' : self._name,
                              'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
                              'URL' : self._cfg.getURL(),
                              'messageSender' : MessageSender( self._name, self._msgBroker ),
                              'validNames' : self._validNames,
                              'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
                            }
    #Call static initialization function
    try:
      self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
                                                     self._lockManager,
                                                     self._msgBroker,
                                                     self._monitor )
      if self._handler[ 'init' ]:
        for initFunc in self._handler[ 'init' ]:
          gLogger.verbose( "Executing initialization function" )
          try:
            result = initFunc( dict( self._serviceInfoDict ) )
          except Exception as excp:
            gLogger.exception( "Exception while calling initialization function", lException = excp )
            return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
          if not isReturnStructure( result ):
            return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
          if not result[ 'OK' ]:
            return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
    except Exception as e:
      errMsg = "Exception while initializing %s" % self._name
      gLogger.exception( e )
      gLogger.exception( errMsg )
      return S_ERROR( errMsg )

    #Load actions after the handler has initialized itself
    result = self._loadActions()
    if not result[ 'OK' ]:
      return result
    self._actions = result[ 'Value' ]

    gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )

    return S_OK()

  def __init__( self, **kwargs ):
    """ Constructor  
    """
    if 'hosts' in kwargs:
      self.__hosts = kwargs['hosts']
      del kwargs['hosts']
    else:  
      result = Registry.getHosts()
      if result['OK']:
        self.__hosts = result['Value']
      else:
        self.__hosts = []
      # Excluded hosts
      if 'exclude' in kwargs:
        self.__hosts = list ( set( self.__hosts ) - set( kwargs[ 'exclude' ] ) )

    # Ping the hosts to remove those that don't have a SystemAdministrator service
    sysAdminHosts = []
    for host in self.__hosts:
      client = SystemAdministratorClient( host )
      result = client.ping()
      if result[ 'OK' ]:
        sysAdminHosts.append( host )
    self.__hosts = sysAdminHosts
      
    self.__kwargs = dict( kwargs )  
    self.__pool = ThreadPool( len( self.__hosts ) )  
    self.__resultDict = {}

  def initialize( self ):
    """ Standard constructor
    """

    try:
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

      self.StorageElementToBeChecked = Queue.Queue()
      self.StorageElementInCheck     = []

      self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
      self.threadPool         = ThreadPool( self.maxNumberOfThreads,
                                            self.maxNumberOfThreads )

      if not self.threadPool:
        self.log.error( 'Can not create Thread Pool' )
        return S_ERROR( 'Can not create Thread Pool' )

      self.setup                = getSetup()[ 'Value' ]
      self.VOExtension          = getExt()
      self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ]
      self.nc                   = NotificationClient()
      self.diracAdmin           = DiracAdmin()
      self.csAPI                = CSAPI()

      for _i in xrange( self.maxNumberOfThreads ):
        self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )

      return S_OK()

    except Exception:
      errorStr = "StElWriteInspectorAgent initialization"
      gLogger.exception( errorStr )
      return S_ERROR( errorStr )

  def initialize( self ):

    # Attribute defined outside __init__ 
    # pylint: disable-msg=W0201

    try:
      self.rsClient             = ResourceStatusClient()
      self.resourcesFreqs       = CS.getTypedDictRootedAtOperations( 'CheckingFreqs/ResourcesFreqs' )
      self.resourcesToBeChecked = Queue.Queue()
      self.resourceNamesInCheck = []

      self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
      self.threadPool         = ThreadPool( self.maxNumberOfThreads,
                                            self.maxNumberOfThreads )
      if not self.threadPool:
        self.log.error( 'Can not create Thread Pool' )
        return S_ERROR( 'Can not create Thread Pool' )

      for _i in xrange( self.maxNumberOfThreads ):
        self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )

      return S_OK()

    except Exception:
      errorStr = "RSInspectorAgent initialization"
      self.log.exception( errorStr )
      return S_ERROR( errorStr )

  def initialize(self):
    """ standard initialize
    """
    # few parameters
    self.pluginLocation = self.am_getOption('PluginLocation',
                                            'DIRAC.TransformationSystem.Agent.TransformationPlugin')
    self.transformationStatus = self.am_getOption('transformationStatus', ['Active', 'Completing', 'Flush'])
    # Prepare to change the name of the CS option as MaxFiles is ambiguous
    self.maxFiles = self.am_getOption('MaxFilesToProcess', self.am_getOption('MaxFiles', 5000))

    agentTSTypes = self.am_getOption('TransformationTypes', [])
    if agentTSTypes:
      self.transformationTypes = sorted(agentTSTypes)
    else:
      dataProc = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
      dataManip = Operations().getValue('Transformations/DataManipulation', ['Replication', 'Removal'])
      self.transformationTypes = sorted(dataProc + dataManip)

    # clients
    self.transfClient = TransformationClient()

    # for caching using a pickle file
    self.workDirectory = self.am_getWorkDirectory()
    self.cacheFile = os.path.join(self.workDirectory, 'ReplicaCache.pkl')
    self.controlDirectory = self.am_getControlDirectory()

    # remember the offset if any in TS
    self.lastFileOffset = {}

    # Validity of the cache
    self.replicaCache = {}
    self.replicaCacheValidity = self.am_getOption('ReplicaCacheValidity', 2)

    self.noUnusedDelay = self.am_getOption('NoUnusedDelay', 6)

    # Get it threaded
    maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
    threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
    self.log.info("Multithreaded with %d threads" % maxNumberOfThreads)

    for i in xrange(maxNumberOfThreads):
      threadPool.generateJobAndQueueIt(self._execute, [i])

    self.log.info("Will treat the following transformation types: %s" % str(self.transformationTypes))

    return S_OK()

  def initialize( self ):
    """ standard initialize
    """
    # few parameters
    self.pluginLocation = self.am_getOption( 'PluginLocation',
                                             'DIRAC.TransformationSystem.Agent.TransformationPlugin' )
    self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush'] )
    self.maxFiles = self.am_getOption( 'MaxFiles', 5000 )

    agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
    if agentTSTypes:
      self.transformationTypes = sorted( agentTSTypes )
    else:
      dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
      dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
      self.transformationTypes = sorted( dataProc + dataManip )



    # clients
    self.transfClient = TransformationClient()

    # shifter
    self.am_setOption( 'shifterProxy', 'ProductionManager' )

    # for caching using a pickle file
    self.__readCache()
    self.workDirectory = self.am_getWorkDirectory()
    self.cacheFile = os.path.join( self.workDirectory, 'ReplicaCache.pkl' )
    self.controlDirectory = self.am_getControlDirectory()
    self.replicaCacheValidity = self.am_getOption( 'ReplicaCacheValidity', 2 )
    self.noUnusedDelay = self.am_getOption( 'NoUnusedDelay', 6 )
    self.dateWriteCache = datetime.datetime.utcnow()

    # Get it threaded
    maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
    threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
    self.log.info( "Multithreaded with %d threads" % maxNumberOfThreads )

    for i in xrange( maxNumberOfThreads ):
      threadPool.generateJobAndQueueIt( self._execute, [i] )

    self.log.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) )

    return S_OK()

  def initialize( self ):
    """ standard initialize
    """

    self.__readCache()
    self.dateWriteCache = datetime.datetime.utcnow()

    self.am_setOption( 'shifterProxy', 'ProductionManager' )

    # Get it threaded
    maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
    threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
    self.log.info( "Multithreaded with %d threads" % maxNumberOfThreads )

    for i in xrange( maxNumberOfThreads ):
      threadPool.generateJobAndQueueIt( self._execute, [i] )

    return S_OK()

  def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
               WMSAdminIn = None):
    """
    Standard constructor

    :params:
      :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')

      :attr:`rsDBIn`: optional ResourceStatusDB object
      (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)

      :attr:`commandCallerIn`: optional CommandCaller object
      (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)

      :attr:`infoGetterIn`: optional InfoGetter object
      (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)

      :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
      (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
    """

    self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)

    if rsDBIn is not None:
      self.rsDB = rsDBIn
    else:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
    self.rmDB = ResourceManagementDB()

    if commandCallerIn is not None:
      self.cc = commandCallerIn
    else:
      from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
      self.cc = CommandCaller()

    if infoGetterIn is not None:
      self.ig = infoGetterIn
    else:
      from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
      self.ig = InfoGetter(VOExtension)

    if WMSAdminIn is not None:
      self.WMSAdmin = WMSAdminIn
    else:
      from DIRAC.Core.DISET.RPCClient import RPCClient
      self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")

    self.threadPool = ThreadPool( 2, 5 )

    self.lockObj = threading.RLock()

    self.infoForPanel_res = {}

    def __addPool(self, poolName):
        """
      create a new thread Pool, by default it has 2 executing threads and 40 requests
      in the Queue
    """

        if not poolName:
            return None
        if poolName in self.pools:
            return None
        pool = ThreadPool(
            self.am_getOption("minThreadsInPool"),
            self.am_getOption("maxThreadsInPool"),
            self.am_getOption("totalThreadsInPool"),
        )
        # Daemonize except "Default" pool
        if poolName != "Default":
            pool.daemonize()
        self.pools[poolName] = pool
        return poolName

 def initialize( self ):
   """ agent's initialisation """
   self.transferDB = TransferDB()
   self.am_setOption( "shifterProxy", "DataManager" )
   self.minThreads = self.am_getOption( "MinThreads", self.minThreads )
   self.maxThreads = self.am_getOption( "MaxThreads", self.maxThreads )
   minmax = ( abs( self.minThreads ), abs( self.maxThreads ) )
   self.minThreads, self.maxThreads = min( minmax ), max( minmax )
   self.log.info( "ThreadPool min threads = %s" % self.minThreads )
   self.log.info( "ThreadPool max threads = %s" % self.maxThreads )
   self.threadPool = ThreadPool( self.minThreads, self.maxThreads )
   self.threadPool.daemonize()
   return S_OK()

  def initialize(self):

    requiredLifeTime = self.am_getOption( "MinimumLifeTime", 3600 )
    renewedLifeTime = self.am_getOption( "RenewedLifeTime", 54000 )
    self.proxyDB = ProxyDB( useMyProxy = True )

    gLogger.info( "Minimum Life time      : %s" % requiredLifeTime )
    gLogger.info( "Life time on renew     : %s" % renewedLifeTime )
    gLogger.info( "MyProxy server         : %s" % self.proxyDB.getMyProxyServer() )
    gLogger.info( "MyProxy max proxy time : %s" % self.proxyDB.getMyProxyMaxLifeTime() )

    self.__threadPool = ThreadPool( 1, 10 )
    return S_OK()