本文整理汇总了Python中distributed.Center类的典型用法代码示例。如果您正苦于以下问题:Python Center类的具体用法?Python Center怎么用?Python Center使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Center类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_monitor_resources
def test_monitor_resources():
pytest.importorskip('psutil')
c = Center(ip='127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
assert n.process.is_alive()
d = n.resource_collect()
assert {'cpu_percent', 'memory_percent'}.issubset(d)
assert isinstance(d['timestamp'], datetime)
stream = yield connect(ip=n.ip, port=n.port)
yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})
for i in range(3):
msg = yield read(stream)
assert isinstance(msg, dict)
assert {'cpu_percent', 'memory_percent'}.issubset(msg)
stream.close()
yield n._close()
c.stop()
示例2: test_nanny
def test_nanny():
c = Center('127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start(0)
nn = rpc(ip=n.ip, port=n.port)
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.worker_services[n.worker_address]['nanny'] > 1024
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.worker_services
assert not n.process
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.worker_services
assert not n.process
yield nn.instantiate()
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.worker_services[n.worker_address]['nanny'] > 1024
yield nn.terminate()
assert not n.process
if n.process:
n.process.terminate()
yield n._close()
c.stop()
示例3: g
def g():
c = Center('127.0.0.1')
c.listen(0)
a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker(c.ip, c.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(c.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(c, a, b)
except Exception as e:
logger.exception(e)
raise
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
c.stop()
示例4: test_monitor_resources
def test_monitor_resources(loop):
c = Center('127.0.0.1', 8026)
n = Nanny('127.0.0.1', 8027, 8028, '127.0.0.1', 8026, ncores=2)
c.listen(c.port)
@gen.coroutine
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
assert n.process.is_alive()
d = n.resource_collect()
assert {'cpu_percent', 'memory_percent'}.issubset(d)
assert isinstance(d['timestamp'], datetime)
stream = yield connect(ip=n.ip, port=n.port)
yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})
for i in range(3):
msg = yield read(stream)
assert isinstance(msg, dict)
assert {'cpu_percent', 'memory_percent'}.issubset(msg)
stream.close()
yield n._close()
c.stop()
loop.run_sync(f)
示例5: test_fast_kill
def test_fast_kill(loop):
from distributed import Nanny, rpc
c = Center('127.0.0.1', 8006)
a = Nanny('127.0.0.1', 8007, 8008, '127.0.0.1', 8006, ncores=2)
b = Nanny('127.0.0.1', 8009, 8010, '127.0.0.1', 8006, ncores=2)
e = Executor((c.ip, c.port), start=False, loop=loop)
c.listen(c.port)
@gen.coroutine
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
yield e._start()
L = e.map(sleep, range(10))
try:
start = time()
yield e._restart()
assert time() - start < 5
assert all(x.status == 'cancelled' for x in L)
x = e.submit(inc, 1)
result = yield x._result()
assert result == 2
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
loop.run_sync(f)
示例6: test_nanny_process_failure
def test_nanny_process_failure(loop):
c = Center('127.0.0.1', 8036)
n = Nanny('127.0.0.1', 8037, 8038, '127.0.0.1', 8036, ncores=2)
c.listen(c.port)
@gen.coroutine
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data={'x': 1, 'y': 2})
with ignoring(StreamClosedError):
yield ww.compute(function=sys.exit, args=(0,), key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores:
yield gen.sleep(0.01)
assert time() - start < 2
yield n._close()
c.stop()
loop.run_sync(f)
示例7: run_center
def run_center(port):
from distributed import Center
from tornado.ioloop import IOLoop
import logging
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1', port)
center.listen(port)
IOLoop.current().start()
IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
示例8: main
def main(host, port):
if host is None:
host = get_ip()
logger.info("Start center at %s:%d", host, port)
center = Center(host)
center.listen(port)
IOLoop.current().start()
IOLoop.current().close()
logger.info("\nEnd center at %s:%d", host, port)
示例9: run_center
def run_center(port):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1', port)
center.listen(port)
loop.start()
示例10: test_restart
def test_restart(loop):
from distributed import Nanny, rpc
c = Center('127.0.0.1', 8006)
a = Nanny('127.0.0.1', 8007, 8008, '127.0.0.1', 8006, ncores=2)
b = Nanny('127.0.0.1', 8009, 8010, '127.0.0.1', 8006, ncores=2)
c.listen(c.port)
@gen.coroutine
def f():
yield a._start()
yield b._start()
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
f = yield e._restart()
assert f is e
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
loop.run_sync(f)
示例11: g
def g():
c = Center('127.0.0.1', 8017)
c.listen(c.port)
a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=2)
yield a._start()
b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1)
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
yield f(c, a, b)
finally:
with ignoring():
yield a._close()
with ignoring():
yield b._close()
c.stop()
示例12: test_restart
def test_restart():
from distributed import Nanny, rpc
c = Center('127.0.0.1')
c.listen(0)
a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield [a._start(), b._start()]
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
f = yield e._restart()
assert f is e
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
示例13: run_center
def run_center(q):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1')
while True:
try:
center.listen(0)
break
except Exception as e:
logging.info("Could not start center on port. Retrying",
exc_info=True)
q.put(center.port)
loop.start()
示例14: test_nanny_process_failure
def test_nanny_process_failure():
c = Center('127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
first_dir = n.worker_dir
assert os.path.exists(first_dir)
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
with ignoring(StreamClosedError):
yield ww.compute(function=dumps(sys.exit),
args=dumps((0,)),
key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 2
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
nn.close_streams()
c.stop()
示例15: test_monitor_resources
def test_monitor_resources():
pytest.importorskip('psutil')
c = Center('127.0.0.1')
c.listen(0)
a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
s = Scheduler((c.ip, c.port), resource_interval=0.01, resource_log_size=3)
yield a._start()
yield b._start()
yield s.sync_center()
done = s.start()
try:
assert s.ncores == {('127.0.0.1', a.worker_port): 2,
('127.0.0.1', b.worker_port): 2}
assert s.nannies == {(n.ip, n.worker_port): n.port
for n in [a, b]}
while any(len(v) < 3 for v in s.resource_logs.values()):
yield gen.sleep(0.01)
yield gen.sleep(0.1)
assert set(s.resource_logs) == {a.address, b.address}
assert all(len(v) == 3 for v in s.resource_logs.values())
d = s.diagnostic_resources(n=2)
assert set(d) == {a.worker_address, b.worker_address}
assert set(d[a.worker_address]).issubset({'cpu', 'memory', 'time'})
assert all(len(v) == 2 for v in d[a.worker_address].values())
s.put({'op': 'close'})
yield done
finally:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield a._close(timeout=0.5)
with ignoring(TimeoutError, StreamClosedError, OSError):
yield b._close(timeout=0.5)
c.stop()