Python ThreadPool.apply_async方法代码示例

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import apply_async [as 别名]
class Engine(BaseEngine, Signaler):

    #
    # 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
    # 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
    #

    def __init__(self, server):
        self._server = server
        self._pool = ThreadPool(CPUS * 4)
        self._listen_sock = None
        self._wsgi_server = None

        BaseEngine.__init__(self, server)
        Signaler.__init__(self)

    def run(self):

        from engine.config import HOST, PORT

        self._listen_sock = socket(family=AF_INET)
        self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
        self._listen_sock.bind((HOST, PORT))
        self._listen_sock.listen(2048)
        self._listen_sock.setblocking(0)

        self.fork_workers(WORKERS or CPUS + 1)
        self.parent_execute()

    def fork_workers(self, num):
        for i in range(num):
            if fork() == 0:
                self.worker_execute()
                exit(0)

    def worker_execute(self):
        Signaler.worker_execute(self)

        # 启动服务器。
        kwargs = HTTPS and \
            {k: app_path("ssl/" + v) for k, v in (("keyfile", HTTPS_KEY), ("certfile", HTTPS_CERT))} or \
            {}

        self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs)
        self._wsgi_server.serve_forever()

        # 等待所有处理结束，超时 10 秒。
        hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)

    def worker_stop(self, graceful):
        stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
        graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop()

    def async_execute(self, func, *args, **kwargs):
        e = Event()
        g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set())
        e.wait()
        return g.get()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import apply_async [as 别名]
class TestPool(TestCase):
    __timeout__ = 5
    size = 1

    def setUp(self):
        greentest.TestCase.setUp(self)
        self.pool = ThreadPool(self.size)

    def test_apply(self):
        papply = self.pool.apply
        self.assertEqual(papply(sqr, (5,)), sqr(5))
        self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3))

    def test_map(self):
        pmap = self.pool.map
        self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10))))
        self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100))))

    def test_async(self):
        res = self.pool.apply_async(sqr, (7, TIMEOUT1,))
        get = TimingWrapper(res.get)
        self.assertEqual(get(), 49)
        self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)

    def test_async_callback(self):
        result = []
        res = self.pool.apply_async(sqr, (7, TIMEOUT1,), callback=lambda x: result.append(x))
        get = TimingWrapper(res.get)
        self.assertEqual(get(), 49)
        self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
        gevent.sleep(0)  # let's the callback run
        assert result == [49], result

    def test_async_timeout(self):
        res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2))
        get = TimingWrapper(res.get)
        self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2)
        self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1)
        self.pool.join()

    def test_imap(self):
        it = self.pool.imap(sqr, range(10))
        self.assertEqual(list(it), list(map(sqr, range(10))))

        it = self.pool.imap(sqr, range(10))
        for i in range(10):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

        it = self.pool.imap(sqr, range(1000))
        for i in range(1000):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

    def test_imap_random(self):
        it = self.pool.imap(sqr_random_sleep, range(10))
        self.assertEqual(list(it), list(map(sqr, range(10))))

    def test_imap_unordered(self):
        it = self.pool.imap_unordered(sqr, range(1000))
        self.assertEqual(sorted(it), list(map(sqr, range(1000))))

        it = self.pool.imap_unordered(sqr, range(1000))
        self.assertEqual(sorted(it), list(map(sqr, range(1000))))

    def test_imap_unordered_random(self):
        it = self.pool.imap_unordered(sqr_random_sleep, range(10))
        self.assertEqual(sorted(it), list(map(sqr, range(10))))

    def test_terminate(self):
        result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2))
        gevent.sleep(0.1)
        kill = TimingWrapper(self.pool.kill)
        kill()
        assert kill.elapsed < 0.5, kill.elapsed
        result.join()

    def sleep(self, x):
        sleep(float(x) / 10.)
        return str(x)

    def test_imap_unordered_sleep(self):
        # testing that imap_unordered returns items in competion order
        result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2]))
        if self.pool.size == 1:
            expected = ['10', '1', '2']
        else:
            expected = ['1', '2', '10']
        self.assertEqual(result, expected)

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import apply_async [as 别名]
class Engine(BaseEngine, Signaler):

    #
    # 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
    # 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
    #

    def __init__(self, server):
        self._server = server
        self._pool = ThreadPool(settings.CPUS * 4)
        self._listen_sock = None
        self._wsgi_server = None

        BaseEngine.__init__(self, server)
        Signaler.__init__(self)

    def run(self):
        if settings.NEED_PATCH_SOCKET_SSL:
            # gevent.pywsgi启动
            from gevent.monkey import patch_socket, patch_ssl
            patch_socket()
            # 在patch socket之后，如果使用https会出错，需要连ssl也patch掉
            patch_ssl()

        if not settings.NEED_GEVENT_THREADPOOL:
            def sync(func, *args, **kwargs):
                return func(*args, **kwargs)
            self.async_execute = sync

        self._listen_sock = socket(family=AF_INET)
        self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
        self._listen_sock.bind((settings.HOST, settings.PORT))
        self._listen_sock.listen(2048)
        self._listen_sock.setblocking(0)

        if settings.WORKERS == 1:
            # 只有一个worker 启用单进程运行
            self.serve_forever()
        else:
            self.fork_workers(settings.WORKERS or settings.CPUS + 1)
            self.parent_execute()

    def fork_workers(self, num):
        for i in range(num):
            if fork() == 0:
                self.worker_execute()
                exit(0)

    def worker_execute(self):
        Signaler.worker_execute(self)

        self.serve_forever()

        # 等待所有处理结束，超时 10 秒。
        hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)

    def serve_forever(self):
        # 启动服务器。
        kwargs = settings.HTTPS and \
            {k: app_path("ssl/" + v) for k, v in (("keyfile", settings.HTTPS_KEY), ("certfile", settings.HTTPS_CERT))} or \
            {}

        if settings.SUPPORT_WEBSOCKET:
            from geventwebsocket.handler import WebSocketHandler
            kwargs["handler_class"] = WebSocketHandler

        self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs)
        self._wsgi_server.serve_forever()

    def worker_stop(self, graceful):
        stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
        graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop()

    def async_execute(self, func, *args, **kwargs):
        e = Event()
        g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set())
        e.wait()
        return g.get()