Python thread.ThreadPoolExecutor类代码示例

class HttpService(object):
    
    def __init__(self):
        self.__async_executor = ThreadPoolExecutor(max_workers=10)
        self.logger = logging.getLogger(__name__)
        self.__http = Http()
    
    def get(self, request):
        return self.make_request(request, 'GET')
    
    def post(self, request):
        return self.make_request(request, 'POST')
    
    def put(self, request):
        return self.make_request(request, 'PUT')
    
    def delete(self, request):
        return self.make_request(request, 'DELETE')
    
    def make_request(self, request, method):
        future = HttpFuture()
        self.__async_executor.submit(self.__do_request, request, method, future)
        return future

    def __do_request(self, request, method, future):
        try:
            uri = request.url + urllib.parse.urlencode(request.parameters)
            headers, content = self.__http.request(uri, method, request.data, request.headers)
            future.fulfill(headers, content)
        except Exception as ex:
            self.logger.exception("Http __do_request attempt failed with exception")

    def __init__(self):
        # just a big number of workers
        ThreadPoolExecutor.__init__(self, max_workers=100)
        self._queue = collections.deque()  # thread-safe queue of futures
        self._waiting_work = collections.deque()  # tuple (WorkItem, future, set=dependences)

        # (dict id(future) -> set): futures running -> dependences used
        self._sets_in_progress = {}
        self._set_remove = threading.Lock()

class ThreadedPoolExecutor(PoolExecutor):
    '''
    Pooled executor implementation based on a wrapped
    ThreadPoolExecutor object.
    '''
    def __init__(self, context, max_workers=1):
        super(ThreadedPoolExecutor, self).__init__(context)
        self._pool = ThreadPoolExecutor(max_workers)
    
    def execute(self, task):
        self._pool.submit(task.processor)

def test_log_traceback_threaded(caplog):
    @log_traceback
    def f():
        raise Exception()

    e = ThreadPoolExecutor(max_workers=1)
    f = e.submit(f)
    while f.running():
        time.sleep(0.1)

    assert caplog.records()[0].message.endswith(" is about to be started")
    assert caplog.records()[1].message.startswith("Traceback")
    assert caplog.records()[1].message.endswith("Exception\n")

 def __init__(self, name, detector, dataflow, emitter,
              ccd, stage, focus, shiftebeam=MTD_MD_UPD, **kwargs):
     """
     shiftebeam (MTD_*): if MTD_EBEAM_SHIFT, will correct the SEM position using beam shift
      (iow, using emitter.shift). If MTD_MD_UPD, it will just update the
      position correction metadata on the SEM images.
     ccd (Optical detector)
     stage (actuator): the sample stage, just to know when re-alignment is needed
     focus (actuator): the _optical_ focuser, just to know when re-alignment is needed
     focuser (actuator): the _e-beam_ focuser, to allow focusing the image
     """
     if "acq_type" not in kwargs:
         kwargs["acq_type"] = model.MD_AT_EM
     super(AlignedSEMStream, self).__init__(name, detector, dataflow, emitter, **kwargs)
     self._ccd = ccd
     self._stage = stage
     self._focus = focus
     self._shiftebeam = shiftebeam
     self.calibrated = model.BooleanVA(False)  # whether the calibration has been already done
     self._last_pos = stage.position.value.copy()
     self._last_pos.update(focus.position.value)  # last known position of the stage
     stage.position.subscribe(self._onMove)
     focus.position.subscribe(self._onMove)
     self._executor = ThreadPoolExecutor(max_workers=1)
     self._beamshift = None

    def __init__(self):
        # Scene processors.
        self.__sceneController = SceneController.instance()
        self.__methodController = MethodController.instance()

        self.__sceneUpdateThreadPool = ThreadPoolExecutor(max_workers=1)
        self.__sceneExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_SCENE_EXEC_THREAD_SIZE)
        self.__sceneExecutionResultThreadPool = ThreadPoolExecutor(max_workers=5)

        # Scene run workers.
        self.__sceneExecLocks = {}
        
        # Favorite edit lock
        self.__fav_lock = threading.Lock()

        # Listeners.
        GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed)
        self.__methodController.listen_to_method_status_change(self.__on_method_status_changed)

    def __init__(self, backbone, brain=None):
        self._backbone = backbone
        self._brain = brain if brain is not None else Brain()
        self._brain_lock = threading.Lock()
        self._regex_to_response = OrderedDict()
        self._scripts = OrderedDict()

        self._pool = ThreadPoolExecutor(max_workers=4)
        self._futures = []  # list of futures submitted to the pool
        self._stop_loop = threading.Event()

 def __init__(self, server_address, RequestHandlerClass,
              bind_and_activate=True, handlers=[],
              srv_path=".",
              configuration={}):
     HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
     self.handlers = sorted(handlers, key=lambda k: k["weight"])
     self.srv_path = srv_path
     self.configuration = configuration
     self.logger = self.setup_logger()
     self.executor = ThreadPoolExecutor(max_workers=20)
     self.initialize_server()

    def __init__(self, rabbitmq_url=None, queue=None, routing_key=None,
                 exchange="message", exchange_type="direct", log=None,
                 max_tasks=5, logging=None):
        """

        == Config dict structure (case adjusted to json configuration):
        {
            "rabbit": {
                "url": "apmq://rabbit",
                "queue": "test",
                "routingKey": "example.json"
                "exchange": "message", // optional, default: message
                "exchangeType:" "topic" // optional, default: topic
            }
        }

        :param str rabbitmq_url: optional url to rabbitmq
        :param str queue: name of the queue
        :param str routing_key: routing key for queue
        :param str exchange: name of the exchange
        :param str exchange_type: type of the exchange
        :param dict config: Manager configuration from parsed json config all
                            the above options can be configured from it
        :param logging.Logger log: optional logger that will replace new one
        :raises exceptions.NotConfigured:
        :return:
        """

        if queue is None:
            raise exceptions.NotConfigured("Misssing queue")

        self._connection = None
        self._channel = None
        self._closing = False
        self._consumer_tag = None
        self._max_tasks = max_tasks  # 2 cores + 1
        self._tasks_number = 0
        self._executor = ThreadPoolExecutor(max_workers=self._max_tasks)
        self._max_tasks_warning_counter = 0

        self._rabbitmq_url = rabbitmq_url
        self._queue = queue
        self._routing_key = routing_key
        self._exchange = exchange
        self._exchange_type = exchange_type

        if log is None:
            from toddler.logging import setup_logging
            if logging is not None:
                self.log = setup_logging(config=logging)
            else:
                self.log = setup_logging()
        else:
            self.log = log

 def __init__(self):
     # Rule processors.
     self.__ruleController = RuleController()
     self.__methodController = MethodController()
     self.__triggerController = TriggerController.instance()
     
     self.__ruleUpdateThreadPool = ThreadPoolExecutor(max_workers=1)
     self.__ruleExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_RULE_EXEC_THREAD_SIZE)
     
     # Rule run workers.
     self.__ruleExecInfos = {}
     self.__condCallGroup = MethodCallGroup()
     self.__execCallGroup = MethodCallGroup()
     
     # Listeners.
     self.__ruleController.listen_to_rule_status_change(self.__on_rule_status_changed)
     GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed)
     self.__methodController.listen_to_method_status_change(self.__on_method_status_changed)
     EventController.instance().listen_to_event_callback(self.__on_method_event_callback)
     self.__triggerController.listen_to_trigger_callback(self.__on_trigger_callback)

 def __init__(self, max_workers=5):
     '''
     This is multi-threaded caller.
     max_workers - Maximum simultaneous threads.
     async - Set it to True then "submit" will not wait for the response.
     '''
     self.__callId_callbackList = {} # 1-level dict, [callId] -- [callback tasks]
     self.__fs_callbackList = {} # 1-level dict, [fs] = [callback tasks]
     self.__fs_callId = {}
     self.__threadPool = ThreadPoolExecutor(max_workers=max_workers)
     self.__lock = Lock()
     self.__taskId = 0

 def __init__(self):
     self.__lock = threading.Lock()
     self.__serLock = SERLock()
     self.__serController = SceneExecutionResultController()
     self.__serRetryThreadPool = ThreadPoolExecutor(max_workers=3)
     
     def after_ser_deleted(serId):
         self.__serLock.declare_as_deleted(serId) # Report serId is deleted to SERLock instance. 
         self.__broadcast_message__ser_deleted(serId) # Broadcast event to mobile client.
     
     SceneExecutionResultDataHandler.AFTER_RECORD_DELETE = after_ser_deleted
     SceneExecutionResultService.__INSTANCE = self

def prepareServer(RequestHandlerClass, pipe, threads, timeout):
    '''
    Prepare in a process the request handling.
    '''
    def process(request, address):
        RequestHandlerClass(request, address, None)
        try: request.shutdown(socket.SHUT_WR)
        except socket.error: pass  # some platforms may raise ENOTCONN here
        request.close()
    
    pool = ThreadPoolExecutor(threads)
    while True:
        if not pipe.poll(timeout): break
        else:
            data = pipe.recv()
            if data is None: break
            elif data is True: continue
            
            requestfd, address = data
            request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET, socket.SOCK_STREAM)
            
            pool.submit(process, request, address)
            
    pool.shutdown(False)

class DispatcherHTTPServer(HTTPServer):
    def __init__(self, server_address, RequestHandlerClass,
                 bind_and_activate=True, handlers=[],
                 srv_path=".",
                 configuration={}):
        HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
        self.handlers = sorted(handlers, key=lambda k: k["weight"])
        self.srv_path = srv_path
        self.configuration = configuration
        self.logger = self.setup_logger()
        self.executor = ThreadPoolExecutor(max_workers=20)
        self.initialize_server()
        
    def initialize_server(self):
        self.logger.info('Initializing server')
    
    def finish_request(self, request, client_address):
        def async_finish_request(server, request, client_address, logger):
            server.RequestHandlerClass(request, client_address, server, logger)
        self.executor.submit(async_finish_request(self, request, client_address, self.logger))
    
    def setup_logger(self):
        logger = None
        if self.configuration.get('log_config_file') is not None:
            logger = self.get_logger(self.configuration.get('log_config_file'))
        else:
            logger = self.get_default_logger()
        return logger
        
    def get_logger(self, config_file):
        logging.config.fileConfig(config_file)
        return logging.getLogger('srv')
    
    def get_default_logger(self):
        logging.basicConfig(level=logging.INFO)
        return logging.getLogger('srv')

 def __init__(self, name, daemon):
     model.Component.__init__(self, name=name, daemon=daemon)
     self.executor = ThreadPoolExecutor(max_workers=1)
     self.number_futures = 0
     self.startAcquire = model.Event() # triggers when the acquisition of .data starts
     self.data = FakeDataFlow(sae=self.startAcquire)
     self.datas = SynchronizableDataFlow()
     
     self.data_count = 0
     self._df = None
     
     # TODO automatically register the property when serializing the Component
     self.prop = model.IntVA(42)
     self.cont = model.FloatContinuous(2.0, [-1, 3.4], unit="C")
     self.enum = model.StringEnumerated("a", set(["a", "c", "bfds"]))
     self.cut = model.IntVA(0, setter=self._setCut)
     self.listval = model.ListVA([2, 65])