Python ThreadPool.dismissWorkers方法代码示例

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import dismissWorkers [as 别名]
    def gits_download(self, url, output="/tmp", threads=20):

        if not self.output:
            self.output = output

        results = self.index(url, output=output)

        if not results:
            return

        args = [((i[0], i[1]), {}) for i in self.giturls]

        # ... and build a WorkRequest object for each item in data
        requests = makeRequests(self.callback,
                                args,
                                self.print_result,
                                self.handle_exception)

        main = ThreadPool(threads)

        for req in requests:
            main.putRequest(req)
            print "Work request #%s added." % req.requestID

        i = 0
        while True:
            try:
                main.poll()
                print "Main thread working...",
                print "(active worker threads: %i)" % (
                    threading.activeCount()-1, )
                if i == 10:
                    print "**** Adding 3 more worker threads..."
                    main.createWorkers(3)
                if i == 20:
                    print "**** Dismissing 2 worker threads..."
                    main.dismissWorkers(2)
                i += 1
            except KeyboardInterrupt:
                print "**** Interrupted!"
                break
            except NoResultsPending:
                print "**** No pending results."
                break
        if main.dismissedWorkers:
            print "Joining all dismissed worker threads..."
            main.joinAllDismissedWorkers()

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import dismissWorkers [as 别名]
class TaskPool(object):

    def __init__(self, limit, logger=None, **kwargs):
        self.limit = limit
        self.logger = logger or log.get_default_logger()
        self._pool = None

    def start(self):
        self._pool = ThreadPool(self.limit)

    def stop(self):
        self._pool.dismissWorkers(self.limit, do_join=True)

    def apply_async(self, target, args=None, kwargs=None, callbacks=None,
            errbacks=None, accept_callback=None, **compat):
        args = args or []
        kwargs = kwargs or {}
        callbacks = callbacks or []
        errbacks = errbacks or []

        on_ready = partial(self.on_ready, callbacks, errbacks)

        self.logger.debug("ThreadPool: Apply %s (args:%s kwargs:%s)" % (
            target, args, kwargs))

        req = WorkRequest(do_work, (target, args, kwargs, on_ready,
                                    accept_callback))
        self._pool.putRequest(req)
        # threadpool also has callback support,
        # but for some reason the callback is not triggered
        # before you've collected the results.
        # Clear the results (if any), so it doesn't grow too large.
        self._pool._results_queue.queue.clear()
        return req

    def on_ready(self, callbacks, errbacks, ret_value):
        """What to do when a worker task is ready and its return value has
        been collected."""

        if isinstance(ret_value, ExceptionInfo):
            if isinstance(ret_value.exception, (
                    SystemExit, KeyboardInterrupt)):    # pragma: no cover
                raise ret_value.exception
            [errback(ret_value) for errback in errbacks]
        else:
            [callback(ret_value) for callback in callbacks]

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import dismissWorkers [as 别名]
def run_example():
    num_workers = 3
    pool = ThreadPool(num_workers)

    # This variable will tell us whether all threads worked or not. Stored in
    # an object (list) otherwise the inner definition cannot modify it.
    success = [True]

    # The exception handler is not required, but if it's not used the error
    # will be silent.
    def exc_handler(work_request, exc_info):
        mldb.log(traceback.format_tb(exc_info[2]))
        exception_type = exc_info[0]
        exception_message = exc_info[1]
        mldb.log(str(exception_type) + ': ' + str(exception_message))
        success[0] = False

        # If there is an error, stop all threads as soon as possible
        pool.dismissWorkers(num_workers, do_join=True)

    # makeRequests takes, as a second argument, a list of tuples where the
    # first element is *args and the second one is **kwargs and where each
    # tuple represents a job to run.
    #
    # Here we schedule two jobs.
    requests = makeRequests(some_func, [(['patate'], {}), (['orange'], {})],
                            exc_callback=exc_handler)
    [pool.putRequest(req) for req in requests]

    # pool.wait will raise an exception if an error occurs in an early jobs and
    # more jobs need to be run. It's ok, if there is an error we want to stop
    # anyway.
    pool.wait()

    # If the error occurred in one of the last jobs pool.wait will have worked
    # so we need to check it anyway.
    if not success[0]:
        mldb.log("An error occured")
        return

    # It is important (MLDBFB-470) to properly dismiss the workers
    pool.dismissWorkers(num_workers, do_join=True)
    mldb.log("Out of main thread")

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import dismissWorkers [as 别名]
class TaskPool(BasePool):

    def on_start(self):
        self._pool = ThreadPool(self.limit)

    def on_stop(self):
        self._pool.dismissWorkers(self.limit, do_join=True)

    def on_apply(self, target, args=None, kwargs=None, callback=None,
            accept_callback=None, **_):
        req = WorkRequest(apply_target, (target, args, kwargs, callback,
                                         accept_callback))
        self._pool.putRequest(req)
        # threadpool also has callback support,
        # but for some reason the callback is not triggered
        # before you've collected the results.
        # Clear the results (if any), so it doesn't grow too large.
        self._pool._results_queue.queue.clear()
        return req

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import dismissWorkers [as 别名]
    # [main.putRequest(req) for req in requests]

    # ...and wait for the results to arrive in the result queue
    # by using ThreadPool.wait(). This would block until results for
    # all work requests have arrived:
    # main.wait()

    # instead we can poll for results while doing something else:
    i = 0
    while True:
        try:
            time.sleep(0.5)
            main.poll()
            print "Main thread working...",
            print "(active worker threads: %i)" % (threading.activeCount()-1, )
            if i == 10:
                print "**** Adding 3 more worker threads..."
                main.createWorkers(3)
            if i == 20:
                print "**** Dismissing 2 worker threads..."
                main.dismissWorkers(2)
            i += 1
        except KeyboardInterrupt:
            print "**** Interrupted!"
            break
        except NoResultsPending:
            print "**** No pending results."
            break
    if main.dismissedWorkers:
        print "Joining all dismissed worker threads..."
        main.joinAllDismissedWorkers()