Python distributed.Worker类代码示例

def get_repl_worker():
    loop = IOLoop.current()
    t = Thread(target=loop.start, daemon=True)
    t.start()
    w = Worker('tcp://127.0.0.1:8786', loop=loop)
    w.start()  # choose randomly assigned port
    return w

    def g():
        c = Center('127.0.0.1')
        c.listen(0)
        a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1')
        yield a._start()
        b = Worker(c.ip, c.port, ncores=1, ip=b_ip)
        yield b._start()

        start = time()
        try:
            while len(c.ncores) < 2:
                yield gen.sleep(0.01)
                if time() - start > 5:
                    raise Exception("Cluster creation timeout")

            yield f(c, a, b)
        except Exception as e:
            logger.exception(e)
            raise
        finally:
            logger.debug("Closing out test cluster")
            for w in [a, b]:
                with ignoring(TimeoutError, StreamClosedError, OSError):
                    yield w._close()
                if os.path.exists(w.local_dir):
                    shutil.rmtree(w.local_dir)
            c.stop()

def test_no_worker_to_memory_restrictions(c, s, a, b):
    x = delayed(slowinc)(1, delay=0.4)
    y = delayed(slowinc)(x, delay=0.4)
    z = delayed(slowinc)(y, delay=0.4)

    yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})

    while not s.tasks:
        yield gen.sleep(0.01)

    w = Worker(s.ip, s.port, ncores=1, name='alice')
    w.put_key_in_memory(y.key, 3)

    yield w._start()

    while len(s.workers) < 3:
        yield gen.sleep(0.01)
    yield gen.sleep(0.3)

    assert s.get_task_status(keys={x.key, y.key, z.key}) == {
        x.key: 'released',
        y.key: 'memory',
        z.key: 'processing',
    }

    yield w._close()

def test_worker_arrives_with_processing_data(c, s, a, b):
    x = delayed(slowinc)(1, delay=0.4)
    y = delayed(slowinc)(x, delay=0.4)
    z = delayed(slowinc)(y, delay=0.4)

    yy, zz = c.persist([y, z])

    while not any(w.processing for w in s.workers.values()):
        yield gen.sleep(0.01)

    w = Worker(s.ip, s.port, ncores=1)
    w.put_key_in_memory(y.key, 3)

    yield w._start()

    start = time()

    while len(s.workers) < 3:
        yield gen.sleep(0.01)

    assert s.get_task_status(keys={x.key, y.key, z.key}) == {
        x.key: 'released',
        y.key: 'memory',
        z.key: 'processing',
    }

    yield w._close()

def test_no_workers_to_memory(c, s):
    x = delayed(slowinc)(1, delay=0.4)
    y = delayed(slowinc)(x, delay=0.4)
    z = delayed(slowinc)(y, delay=0.4)

    yy, zz = c.persist([y, z])

    while not s.tasks:
        yield gen.sleep(0.01)

    w = Worker(s.ip, s.port, ncores=1)
    w.put_key_in_memory(y.key, 3)

    yield w._start()

    start = time()

    while not s.workers:
        yield gen.sleep(0.01)

    assert s.get_task_status(keys={x.key, y.key, z.key}) == {
        x.key: 'released',
        y.key: 'memory',
        z.key: 'processing',
    }

    yield w._close()

    def g():
        s = Scheduler(ip='127.0.0.1')
        done = s.start()
        s.listen(0)
        a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1')
        yield a._start()
        b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip)
        yield b._start()

        start = time()
        try:
            while len(s.ncores) < 2:
                yield gen.sleep(0.01)
                if time() - start > 5:
                    raise Exception("Cluster creation timeout")

            yield f(s, a, b)
        finally:
            logger.debug("Closing out test cluster")
            for w in [a, b]:
                with ignoring(TimeoutError, StreamClosedError, OSError):
                    yield w._close()
                if os.path.exists(w.local_dir):
                    shutil.rmtree(w.local_dir)
            yield s.close()

def test_services(s, a, b):
    c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1',
               services={'http': HTTPWorker})
    yield c._start()
    assert isinstance(c.services['http'], HTTPServer)
    assert c.service_ports['http'] == c.services['http'].port
    assert s.worker_services[c.address]['http'] == c.service_ports['http']

def run_worker(port, center_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop
    import logging
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
    worker.start()
    IOLoop.current().start()
    IOLoop.current().close()  # Never reached. TODO: clean shutdown of IOLoop

def test_scatter_no_workers(s):
    with pytest.raises(gen.TimeoutError):
        yield gen.with_timeout(timedelta(seconds=0.1),
                              s.scatter(data={'x': dumps(1)}, client='alice'))

    w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
    yield [s.scatter(data={'x': dumps(1)}, client='alice'),
           w._start()]

    assert w.data['x'] == 1

def test_add_remove_worker(s):
    events = []

    class MyPlugin(SchedulerPlugin):
        def add_worker(self, worker, scheduler):
            assert scheduler is s
            events.append(('add_worker', worker))

        def remove_worker(self, worker, scheduler):
            assert scheduler is s
            events.append(('remove_worker', worker))

    plugin = MyPlugin()
    s.add_plugin(plugin)
    assert events == []

    a = Worker(s.address)
    b = Worker(s.address)
    yield a._start()
    yield b._start()
    yield a._close()
    yield b._close()

    assert events == [('add_worker', a.address),
                      ('add_worker', b.address),
                      ('remove_worker', a.address),
                      ('remove_worker', b.address),
                      ]

    events[:] = []
    s.remove_plugin(plugin)
    a = Worker(s.address)
    yield a._start()
    yield a._close()
    assert events == []

def run_worker(port, center_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
    worker.start()
    loop.start()

def test_services_port(s, a, b):
    c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1',
               services={('http', 9898): HTTPWorker})
    yield c._start()
    assert isinstance(c.services['http'], HTTPServer)
    assert (c.service_ports['http']
         == c.services['http'].port
         == s.worker_info[c.address]['services']['http']
         == 9898)

    c.services['http'].stop()
    yield c._close()

def run_worker(q, center_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    with log_errors():
        IOLoop.clear_instance()
        loop = IOLoop(); loop.make_current()
        PeriodicCallback(lambda: None, 500).start()
        logging.getLogger("tornado").setLevel(logging.CRITICAL)
        worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
        loop.run_sync(lambda: worker._start(0))
        q.put(worker.port)
        loop.start()

def test_worker_name():
    s = Scheduler()
    s.start(0)
    w = Worker(s.ip, s.port, name='alice')
    yield w._start()
    assert s.worker_info[w.address]['name'] == 'alice'
    assert s.aliases['alice'] == w.address

    with pytest.raises(ValueError):
        w = Worker(s.ip, s.port, name='alice')
        yield w._start()

    yield s.close()
    yield w._close()

def test_scatter_no_workers(c, s):
    with pytest.raises(gen.TimeoutError):
        yield s.scatter(data={'x': 1}, client='alice', timeout=0.1)

    start = time()
    with pytest.raises(gen.TimeoutError):
        yield c.scatter(123, timeout=0.1)
    assert time() < start + 1.5

    w = Worker(s.ip, s.port, ncores=3)
    yield [c.scatter(data={'y': 2}, timeout=5),
           w._start()]

    assert w.data['y'] == 2
    yield w._close()