Python distributed.Scheduler类代码示例

def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler},
                          loop=loop)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
                    tcp_port=port, bokeh_port=bokeh_port,
                    bokeh_whitelist=bokeh_whitelist, show=show)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    if bokeh_proc:
        bokeh_proc.close()

    logger.info("End scheduler at %s:%d", ip, port)

def main(center, host, port, http_port, bokeh_port):
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    try:
        import bokeh
    except ImportError:
        pass
    else:
        import distributed.diagnostics.bokeh
        hosts = ['%s:%d' % (h, bokeh_port) for h in
                 ['localhost', '127.0.0.1', ip, socket.gethostname()]]
        dirname = os.path.dirname(distributed.__file__)
        path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status')
        proc = subprocess.Popen(['bokeh', 'serve', path,
                                 '--log-level', 'warning',
                                 '--port', str(bokeh_port)] +
                                 sum([['--host', host] for host in hosts], []))

        distributed.diagnostics.bokeh.server_process = proc  # monkey patch

        logger.info(" Start Bokeh UI at:        http://%s:%d/status/"
                    % (ip, bokeh_port))

    loop.start()
    loop.close()
    scheduler.stop()

    logger.info("End scheduler at %s:%d", ip, port)

    def g():
        s = Scheduler(ip='127.0.0.1')
        done = s.start()
        s.listen(0)
        a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1')
        yield a._start()
        b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip)
        yield b._start()

        start = time()
        try:
            while len(s.ncores) < 2:
                yield gen.sleep(0.01)
                if time() - start > 5:
                    raise Exception("Cluster creation timeout")

            yield f(s, a, b)
        finally:
            logger.debug("Closing out test cluster")
            for w in [a, b]:
                with ignoring(TimeoutError, StreamClosedError, OSError):
                    yield w._close()
                if os.path.exists(w.local_dir):
                    shutil.rmtree(w.local_dir)
            yield s.close()

def start_cluster(ncores, scheduler_addr, loop, security=None,
                  Worker=Worker, scheduler_kwargs={}, worker_kwargs={}):
    s = Scheduler(loop=loop, validate=True, security=security,
                  **scheduler_kwargs)
    done = s.start(scheduler_addr)
    workers = [Worker(s.address, ncores=ncore[1], name=i, security=security,
                      loop=loop, validate=True,
                      **(merge(worker_kwargs, ncore[2])
                         if len(ncore) > 2
                         else worker_kwargs))
               for i, ncore in enumerate(ncores)]
    for w in workers:
        w.rpc = workers[0].rpc

    yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]

    start = time()
    while (len(s.workers) < len(ncores) or
           any(comm.comm is None for comm in s.stream_comms.values())):
        yield gen.sleep(0.01)
        if time() - start > 5:
            yield [w._close(timeout=1) for w in workers]
            yield s.close(fast=True)
            raise Exception("Cluster creation timeout")
    raise gen.Return((s, workers))

def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    if _bokeh:
        try:
            import bokeh
            import distributed.bokeh
            hosts = ['%s:%d' % (h, bokeh_port) for h in
                     ['localhost', '127.0.0.1', ip, socket.gethostname(),
                      host] + list(bokeh_whitelist)]
            dirname = os.path.dirname(distributed.__file__)
            paths = [os.path.join(dirname, 'bokeh', name)
                     for name in ['status', 'tasks']]
            binname = sys.argv[0][:-len('dscheduler')] + 'bokeh'
            args = ([binname, 'serve'] + paths +
                    ['--log-level', 'warning',
                     '--check-unused-sessions=50',
                     '--unused-session-lifetime=1',
                     '--port', str(bokeh_port)] +
                     sum([['--host', host] for host in hosts], []))
            if show:
                args.append('--show')

            bokeh_options = {'host': host if given_host else '127.0.0.1',
                             'http-port': http_port,
                             'tcp-port': port,
                             'bokeh-port': bokeh_port}
            with open('.dask-web-ui.json', 'w') as f:
                json.dump(bokeh_options, f, indent=2)

            if sys.version_info[0] >= 3:
                from bokeh.command.bootstrap import main
                ctx = multiprocessing.get_context('spawn')
                bokeh_proc = ctx.Process(target=main, args=(args,))
                bokeh_proc.daemon = True
                bokeh_proc.start()
            else:
                bokeh_proc = subprocess.Popen(args)

            logger.info(" Bokeh UI at:  http://%s:%d/status/"
                        % (ip, bokeh_port))
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    bokeh_proc.terminate()

    logger.info("End scheduler at %s:%d", ip, port)

def test_scheduler_file():
    with tmpfile() as fn:
        s = Scheduler(scheduler_file=fn)
        s.start(8008)
        w = Nanny(scheduler_file=fn)
        yield w._start()
        assert set(s.workers) == {w.worker_address}
        yield w._close()
        s.stop()

def main(center, port):
    loop = IOLoop.current()
    scheduler = Scheduler(center, services={'http': HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    done = scheduler.start(port)

    loop.start()
    loop.close()
    scheduler.stop()

    logging.info("End scheduler at %s:%d", ip, port)

def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
         scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port):
    if interface:
        host = get_ip_interface(interface)
    else:
        host = None

    if rank == 0 and scheduler:
        try:
            from distributed.bokeh.scheduler import BokehScheduler
        except ImportError:
            services = {}
        else:
            services = {('bokeh',  bokeh_port): partial(BokehScheduler,
                                                        prefix=bokeh_prefix)}
        scheduler = Scheduler(scheduler_file=scheduler_file,
                              loop=loop,
                              services=services)
        addr = uri_from_host_port(host, None, 8786)
        scheduler.start(addr)
        try:
            loop.start()
            loop.close()
        finally:
            scheduler.stop()
    else:
        W = Nanny if nanny else Worker
        worker = W(scheduler_file=scheduler_file,
                   loop=loop,
                   name=rank if scheduler else None,
                   ncores=nthreads,
                   local_dir=local_directory,
                   services={('bokeh', bokeh_worker_port): BokehWorker},
                   memory_limit=memory_limit)
        addr = uri_from_host_port(host, None, 0)

        @gen.coroutine
        def run():
            yield worker._start(addr)
            while worker.status != 'closed':
                yield gen.sleep(0.2)

        try:
            loop.run_sync(run)
            loop.close()
        finally:
            pass

        @gen.coroutine
        def close():
            yield worker._close(timeout=2)

        loop.run_sync(close)

def start_cluster(ncores, Worker=Worker):
    s = Scheduler(ip='127.0.0.1')
    done = s.start(0)
    workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores]

    yield [w._start() for w in workers]

    start = time()
    while len(s.ncores) < len(ncores):
        yield gen.sleep(0.01)
        if time() - start > 5:
            raise Exception("Cluster creation timeout")
    raise gen.Return((s, workers))

def main(center, host, port):
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, services={'http': HTTPScheduler}, ip=ip)
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    loop.start()
    loop.close()
    scheduler.stop()

    logging.info("End scheduler at %s:%d", ip, port)

def start_cluster(ncores, loop, Worker=Worker):
    s = Scheduler(ip='127.0.0.1', loop=loop)
    done = s.start(0)
    workers = [Worker(s.ip, s.port, ncores=v, ip=k, name=i, loop=loop)
                for i, (k, v) in enumerate(ncores)]

    yield [w._start() for w in workers]

    start = time()
    while len(s.ncores) < len(ncores):
        yield gen.sleep(0.01)
        if time() - start > 5:
            raise Exception("Cluster creation timeout")
    raise gen.Return((s, workers))

def start_cluster(ncores):
    s = Scheduler(ip='127.0.0.1')
    s.listen(0)
    workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores]

    IOLoop.current().add_callback(s.start)
    yield [w._start() for w in workers]

    start = time()
    while len(s.ncores) < len(ncores):
        yield gen.sleep(0.01)
        if time() - start > 5:
            raise Exception("Cluster creation timeout")
    raise gen.Return((s, workers))

    def f(c, a, b):
        s = Scheduler((c.ip, c.port), loop=loop)
        yield s.sync_center()
        done = s.start(0)

        progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False,
                                   interval=0.01)
        yield progress.listen()

        assert progress.status == 'finished'
        check_bar_completed(capsys)

        s.close()
        yield done

def run_scheduler(q, nputs, **kwargs):
    from distributed import Scheduler

    # On Python 2.7 and Unix, fork() is used to spawn child processes,
    # so avoid inheriting the parent's IO loop.
    with pristine_loop() as loop:
        scheduler = Scheduler(validate=True, **kwargs)
        done = scheduler.start('127.0.0.1')

        for i in range(nputs):
            q.put(scheduler.address)
        try:
            loop.start()
        finally:
            loop.close(all_fds=True)

def run_scheduler(q, scheduler_port=0, **kwargs):
    from distributed import Scheduler
    from tornado.ioloop import IOLoop, PeriodicCallback
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()

    scheduler = Scheduler(loop=loop, validate=True, **kwargs)
    done = scheduler.start(scheduler_port)

    q.put(scheduler.port)
    try:
        loop.start()
    finally:
        loop.close(all_fds=True)