Python Center.listen方法代码示例

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_monitor_resources():
    pytest.importorskip('psutil')
    c = Center(ip='127.0.0.1')
    c.listen(0)
    n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')

    yield n._start()
    nn = rpc(ip=n.ip, port=n.port)
    assert n.process.is_alive()
    d = n.resource_collect()
    assert {'cpu_percent', 'memory_percent'}.issubset(d)

    assert isinstance(d['timestamp'], datetime)

    stream = yield connect(ip=n.ip, port=n.port)
    yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})

    for i in range(3):
        msg = yield read(stream)
        assert isinstance(msg, dict)
        assert {'cpu_percent', 'memory_percent'}.issubset(msg)

    stream.close()
    yield n._close()
    c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_nanny():
    c = Center('127.0.0.1')
    c.listen(0)
    n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')

    yield n._start(0)
    nn = rpc(ip=n.ip, port=n.port)
    assert n.process.is_alive()
    assert c.ncores[n.worker_address] == 2
    assert c.worker_services[n.worker_address]['nanny'] > 1024

    yield nn.kill()
    assert n.worker_address not in c.ncores
    assert n.worker_address not in c.worker_services
    assert not n.process

    yield nn.kill()
    assert n.worker_address not in c.ncores
    assert n.worker_address not in c.worker_services
    assert not n.process

    yield nn.instantiate()
    assert n.process.is_alive()
    assert c.ncores[n.worker_address] == 2
    assert c.worker_services[n.worker_address]['nanny'] > 1024

    yield nn.terminate()
    assert not n.process

    if n.process:
        n.process.terminate()

    yield n._close()
    c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
    def g():
        c = Center('127.0.0.1')
        c.listen(0)
        a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1')
        yield a._start()
        b = Worker(c.ip, c.port, ncores=1, ip=b_ip)
        yield b._start()

        start = time()
        try:
            while len(c.ncores) < 2:
                yield gen.sleep(0.01)
                if time() - start > 5:
                    raise Exception("Cluster creation timeout")

            yield f(c, a, b)
        except Exception as e:
            logger.exception(e)
            raise
        finally:
            logger.debug("Closing out test cluster")
            for w in [a, b]:
                with ignoring(TimeoutError, StreamClosedError, OSError):
                    yield w._close()
                if os.path.exists(w.local_dir):
                    shutil.rmtree(w.local_dir)
            c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_monitor_resources(loop):
    c = Center('127.0.0.1', 8026)
    n = Nanny('127.0.0.1', 8027, 8028, '127.0.0.1', 8026, ncores=2)
    c.listen(c.port)

    @gen.coroutine
    def f():
        nn = rpc(ip=n.ip, port=n.port)
        yield n._start()
        assert n.process.is_alive()
        d = n.resource_collect()
        assert {'cpu_percent', 'memory_percent'}.issubset(d)

        assert isinstance(d['timestamp'], datetime)

        stream = yield connect(ip=n.ip, port=n.port)
        yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})

        for i in range(3):
            msg = yield read(stream)
            assert isinstance(msg, dict)
            assert {'cpu_percent', 'memory_percent'}.issubset(msg)

        stream.close()
        yield n._close()
        c.stop()

    loop.run_sync(f)

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_fast_kill(loop):
    from distributed import Nanny, rpc
    c = Center('127.0.0.1', 8006)
    a = Nanny('127.0.0.1', 8007, 8008, '127.0.0.1', 8006, ncores=2)
    b = Nanny('127.0.0.1', 8009, 8010, '127.0.0.1', 8006, ncores=2)
    e = Executor((c.ip, c.port), start=False, loop=loop)
    c.listen(c.port)
    @gen.coroutine
    def f():
        yield a._start()
        yield b._start()

        while len(c.ncores) < 2:
            yield gen.sleep(0.01)
        yield e._start()

        L = e.map(sleep, range(10))

        try:
            start = time()
            yield e._restart()
            assert time() - start < 5

            assert all(x.status == 'cancelled' for x in L)

            x = e.submit(inc, 1)
            result = yield x._result()
            assert result == 2
        finally:
            yield a._close()
            yield b._close()
            yield e._shutdown(fast=True)
            c.stop()

    loop.run_sync(f)

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_nanny_process_failure(loop):
    c = Center('127.0.0.1', 8036)
    n = Nanny('127.0.0.1', 8037, 8038, '127.0.0.1', 8036, ncores=2)
    c.listen(c.port)

    @gen.coroutine
    def f():
        nn = rpc(ip=n.ip, port=n.port)
        yield n._start()

        ww = rpc(ip=n.ip, port=n.worker_port)
        yield ww.update_data(data={'x': 1, 'y': 2})
        with ignoring(StreamClosedError):
            yield ww.compute(function=sys.exit, args=(0,), key='z')

        start = time()
        while n.process.is_alive():  # wait while process dies
            yield gen.sleep(0.01)
            assert time() - start < 2

        start = time()
        while not n.process.is_alive():  # wait while process comes back
            yield gen.sleep(0.01)
            assert time() - start < 2

        start = time()
        while n.worker_address not in c.ncores:
            yield gen.sleep(0.01)
            assert time() - start < 2

        yield n._close()
        c.stop()

    loop.run_sync(f)

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def run_center(port):
    from distributed import Center
    from tornado.ioloop import IOLoop
    import logging
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    center = Center('127.0.0.1', port)
    center.listen(port)
    IOLoop.current().start()
    IOLoop.current().close()  # Never reached. TODO: clean shutdown of IOLoop

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def main(host, port):
    if host is None:
        host = get_ip()

    logger.info("Start center at %s:%d", host, port)
    center = Center(host)
    center.listen(port)
    IOLoop.current().start()
    IOLoop.current().close()
    logger.info("\nEnd center at %s:%d", host, port)

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def run_center(port):
    from distributed import Center
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    center = Center('127.0.0.1', port)
    center.listen(port)
    loop.start()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_restart(loop):
    from distributed import Nanny, rpc
    c = Center('127.0.0.1', 8006)
    a = Nanny('127.0.0.1', 8007, 8008, '127.0.0.1', 8006, ncores=2)
    b = Nanny('127.0.0.1', 8009, 8010, '127.0.0.1', 8006, ncores=2)
    c.listen(c.port)
    @gen.coroutine
    def f():
        yield a._start()
        yield b._start()

        e = Executor((c.ip, c.port), start=False, loop=loop)
        yield e._start()
        assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}

        x = e.submit(inc, 1)
        y = e.submit(inc, x)
        yield y._result()

        cc = rpc(ip=c.ip, port=c.port)
        who_has = yield cc.who_has()
        try:
            assert e.scheduler.who_has == who_has
            assert set(e.scheduler.who_has) == {x.key, y.key}

            f = yield e._restart()
            assert f is e

            assert len(e.scheduler.stacks) == 2
            assert len(e.scheduler.processing) == 2

            who_has = yield cc.who_has()
            assert not who_has
            assert not e.scheduler.who_has

            assert x.cancelled()
            assert y.cancelled()

        finally:
            yield a._close()
            yield b._close()
            yield e._shutdown(fast=True)
            c.stop()

    loop.run_sync(f)

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
    def g():
        c = Center('127.0.0.1', 8017)
        c.listen(c.port)
        a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=2)
        yield a._start()
        b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1)
        yield b._start()

        while len(c.ncores) < 2:
            yield gen.sleep(0.01)

        try:
            yield f(c, a, b)
        finally:
            with ignoring():
                yield a._close()
            with ignoring():
                yield b._close()
            c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_restart():
    from distributed import Nanny, rpc
    c = Center('127.0.0.1')
    c.listen(0)
    a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
    b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')

    yield [a._start(), b._start()]

    e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
    yield e._start()

    assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}

    x = e.submit(inc, 1)
    y = e.submit(inc, x)
    yield y._result()

    cc = rpc(ip=c.ip, port=c.port)
    who_has = yield cc.who_has()
    try:
        assert e.scheduler.who_has == who_has
        assert set(e.scheduler.who_has) == {x.key, y.key}

        f = yield e._restart()
        assert f is e

        assert len(e.scheduler.stacks) == 2
        assert len(e.scheduler.processing) == 2

        who_has = yield cc.who_has()
        assert not who_has
        assert not e.scheduler.who_has

        assert x.cancelled()
        assert y.cancelled()

    finally:
        yield a._close()
        yield b._close()
        yield e._shutdown(fast=True)
        c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def run_center(q):
    from distributed import Center
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    center = Center('127.0.0.1')

    while True:
        try:
            center.listen(0)
            break
        except Exception as e:
            logging.info("Could not start center on port.  Retrying",
                    exc_info=True)

    q.put(center.port)
    loop.start()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_nanny_process_failure():
    c = Center('127.0.0.1')
    c.listen(0)
    n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
    yield n._start()
    nn = rpc(ip=n.ip, port=n.port)
    first_dir = n.worker_dir

    assert os.path.exists(first_dir)

    ww = rpc(ip=n.ip, port=n.worker_port)
    yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
    with ignoring(StreamClosedError):
        yield ww.compute(function=dumps(sys.exit),
                         args=dumps((0,)),
                         key='z')

    start = time()
    while n.process.is_alive():  # wait while process dies
        yield gen.sleep(0.01)
        assert time() - start < 2

    start = time()
    while not n.process.is_alive():  # wait while process comes back
        yield gen.sleep(0.01)
        assert time() - start < 2

    start = time()
    while n.worker_address not in c.ncores or n.worker_dir is None:
        yield gen.sleep(0.01)
        assert time() - start < 2

    second_dir = n.worker_dir

    yield n._close()
    assert not os.path.exists(second_dir)
    assert not os.path.exists(first_dir)
    assert first_dir != n.worker_dir
    nn.close_streams()
    c.stop()

# 需要导入模块: from distributed import Center [as 别名]
# 或者: from distributed.Center import listen [as 别名]
def test_monitor_resources():
    pytest.importorskip('psutil')
    c = Center('127.0.0.1')
    c.listen(0)
    a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
    b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
    s = Scheduler((c.ip, c.port), resource_interval=0.01, resource_log_size=3)

    yield a._start()
    yield b._start()
    yield s.sync_center()
    done = s.start()

    try:
        assert s.ncores == {('127.0.0.1', a.worker_port): 2,
                            ('127.0.0.1', b.worker_port): 2}
        assert s.nannies == {(n.ip, n.worker_port): n.port
                             for n in [a, b]}

        while any(len(v) < 3 for v in s.resource_logs.values()):
            yield gen.sleep(0.01)

        yield gen.sleep(0.1)

        assert set(s.resource_logs) == {a.address, b.address}
        assert all(len(v) == 3 for v in s.resource_logs.values())

        d = s.diagnostic_resources(n=2)
        assert set(d) == {a.worker_address, b.worker_address}
        assert set(d[a.worker_address]).issubset({'cpu', 'memory', 'time'})
        assert all(len(v) == 2 for v in d[a.worker_address].values())

        s.put({'op': 'close'})
        yield done
    finally:
        with ignoring(TimeoutError, StreamClosedError, OSError):
            yield a._close(timeout=0.5)
        with ignoring(TimeoutError, StreamClosedError, OSError):
            yield b._close(timeout=0.5)
        c.stop()