Python Scheduler.stop方法代码示例

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    if _bokeh:
        try:
            import bokeh
            import distributed.bokeh
            hosts = ['%s:%d' % (h, bokeh_port) for h in
                     ['localhost', '127.0.0.1', ip, socket.gethostname(),
                      host] + list(bokeh_whitelist)]
            dirname = os.path.dirname(distributed.__file__)
            paths = [os.path.join(dirname, 'bokeh', name)
                     for name in ['status', 'tasks']]
            binname = sys.argv[0][:-len('dscheduler')] + 'bokeh'
            args = ([binname, 'serve'] + paths +
                    ['--log-level', 'warning',
                     '--check-unused-sessions=50',
                     '--unused-session-lifetime=1',
                     '--port', str(bokeh_port)] +
                     sum([['--host', host] for host in hosts], []))
            if show:
                args.append('--show')

            bokeh_options = {'host': host if given_host else '127.0.0.1',
                             'http-port': http_port,
                             'tcp-port': port,
                             'bokeh-port': bokeh_port}
            with open('.dask-web-ui.json', 'w') as f:
                json.dump(bokeh_options, f, indent=2)

            if sys.version_info[0] >= 3:
                from bokeh.command.bootstrap import main
                ctx = multiprocessing.get_context('spawn')
                bokeh_proc = ctx.Process(target=main, args=(args,))
                bokeh_proc.daemon = True
                bokeh_proc.start()
            else:
                bokeh_proc = subprocess.Popen(args)

            logger.info(" Bokeh UI at:  http://%s:%d/status/"
                        % (ip, bokeh_port))
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    bokeh_proc.terminate()

    logger.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler},
                          loop=loop)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
                    tcp_port=port, bokeh_port=bokeh_port,
                    bokeh_whitelist=bokeh_whitelist, show=show)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    if bokeh_proc:
        bokeh_proc.close()

    logger.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, host, port, http_port, bokeh_port):
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    try:
        import bokeh
    except ImportError:
        pass
    else:
        import distributed.diagnostics.bokeh
        hosts = ['%s:%d' % (h, bokeh_port) for h in
                 ['localhost', '127.0.0.1', ip, socket.gethostname()]]
        dirname = os.path.dirname(distributed.__file__)
        path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status')
        proc = subprocess.Popen(['bokeh', 'serve', path,
                                 '--log-level', 'warning',
                                 '--port', str(bokeh_port)] +
                                 sum([['--host', host] for host in hosts], []))

        distributed.diagnostics.bokeh.server_process = proc  # monkey patch

        logger.info(" Start Bokeh UI at:        http://%s:%d/status/"
                    % (ip, bokeh_port))

    loop.start()
    loop.close()
    scheduler.stop()

    logger.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def test_scheduler_file():
    with tmpfile() as fn:
        s = Scheduler(scheduler_file=fn)
        s.start(8008)
        w = Nanny(scheduler_file=fn)
        yield w._start()
        assert set(s.workers) == {w.worker_address}
        yield w._close()
        s.stop()

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, port):
    loop = IOLoop.current()
    scheduler = Scheduler(center, services={'http': HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    done = scheduler.start(port)

    loop.start()
    loop.close()
    scheduler.stop()

    logging.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
         scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port):
    if interface:
        host = get_ip_interface(interface)
    else:
        host = None

    if rank == 0 and scheduler:
        try:
            from distributed.bokeh.scheduler import BokehScheduler
        except ImportError:
            services = {}
        else:
            services = {('bokeh',  bokeh_port): partial(BokehScheduler,
                                                        prefix=bokeh_prefix)}
        scheduler = Scheduler(scheduler_file=scheduler_file,
                              loop=loop,
                              services=services)
        addr = uri_from_host_port(host, None, 8786)
        scheduler.start(addr)
        try:
            loop.start()
            loop.close()
        finally:
            scheduler.stop()
    else:
        W = Nanny if nanny else Worker
        worker = W(scheduler_file=scheduler_file,
                   loop=loop,
                   name=rank if scheduler else None,
                   ncores=nthreads,
                   local_dir=local_directory,
                   services={('bokeh', bokeh_worker_port): BokehWorker},
                   memory_limit=memory_limit)
        addr = uri_from_host_port(host, None, 0)

        @gen.coroutine
        def run():
            yield worker._start(addr)
            while worker.status != 'closed':
                yield gen.sleep(0.2)

        try:
            loop.run_sync(run)
            loop.close()
        finally:
            pass

        @gen.coroutine
        def close():
            yield worker._close(timeout=2)

        loop.run_sync(close)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, host, port):
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, services={'http': HTTPScheduler}, ip=ip)
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    loop.start()
    loop.close()
    scheduler.stop()

    logging.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(host, port, http_port, bokeh_port, show, _bokeh,
         bokeh_whitelist, prefix, use_xheaders, pid_file):

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)
        atexit.register(del_pid_file)

    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    limit = max(soft, hard // 2)
    resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(ip=ip, loop=loop,
                          services={('http', http_port): HTTPScheduler})
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
                    tcp_port=port, bokeh_port=bokeh_port,
                    bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix,
                    use_xheaders=use_xheaders, quiet=False)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    if bokeh_proc:
        bokeh_proc.close()

    logger.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(center, host, port, http_port, bokeh_port, show, _bokeh):
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    if _bokeh:
        try:
            import bokeh
            import distributed.bokeh
            hosts = ['%s:%d' % (h, bokeh_port) for h in
                     ['localhost', '127.0.0.1', ip, socket.gethostname(), host]]
            dirname = os.path.dirname(distributed.__file__)
            paths = [os.path.join(dirname, 'bokeh', name)
                     for name in ['status', 'tasks']]
            args = (['bokeh', 'serve'] + paths +
                    ['--log-level', 'warning',
                     '--check-unused-sessions=50',
                     '--unused-session-lifetime=1',
                     '--port', str(bokeh_port)] +
                     sum([['--host', host] for host in hosts], []))
            if show:
                args.append('--show')
            from bokeh.command.bootstrap import main
            proc = multiprocessing.Process(target=main, args=(args,))
            proc.daemon = True
            proc.start()

            logger.info(" Start Bokeh UI at:        http://%s:%d/status/"
                        % (ip, bokeh_port))
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    proc.terminate()

    logger.info("End scheduler at %s:%d", ip, port)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,
        use_xheaders, pid_file, scheduler_file, interface,
        local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key):

    enable_proctitle_on_current()
    enable_proctitle_on_children()

    sec = Security(tls_ca_file=tls_ca_file,
                   tls_scheduler_cert=tls_cert,
                   tls_scheduler_key=tls_key,
                   )

    if not host and (tls_ca_file or tls_cert or tls_key):
        host = 'tls://'

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)
        atexit.register(del_pid_file)

    local_directory_created = False
    if local_directory:
        if not os.path.exists(local_directory):
            os.mkdir(local_directory)
            local_directory_created = True
    else:
        local_directory = tempfile.mkdtemp(prefix='scheduler-')
        local_directory_created = True
    if local_directory not in sys.path:
        sys.path.insert(0, local_directory)

    if sys.platform.startswith('linux'):
        import resource   # module fails importing on Windows
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        limit = max(soft, hard // 2)
        resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    if interface:
        if host:
            raise ValueError("Can not specify both interface and host")
        else:
            host = get_ip_interface(interface)

    addr = uri_from_host_port(host, port, 8786)

    loop = IOLoop.current()
    logger.info('-' * 47)

    services = {}
    if _bokeh:
        try:
            from distributed.bokeh.scheduler import BokehScheduler
            services[('bokeh', bokeh_port)] = (BokehScheduler,
                                               {'prefix': bokeh_prefix})
        except ImportError as error:
            if str(error).startswith('No module named'):
                logger.info('Web dashboard not loaded.  Unable to import bokeh')
            else:
                logger.info('Unable to import bokeh: %s' % str(error))

    scheduler = Scheduler(loop=loop, services=services,
                          scheduler_file=scheduler_file,
                          security=sec)
    scheduler.start(addr)
    if not preload:
        preload = dask.config.get('distributed.scheduler.preload')
    if not preload_argv:
        preload_argv = dask.config.get('distributed.scheduler.preload-argv')
    preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv)

    logger.info('Local Directory: %26s', local_directory)
    logger.info('-' * 47)

    install_signal_handlers(loop)

    try:
        loop.start()
        loop.close()
    finally:
        scheduler.stop()
        if local_directory_created:
            shutil.rmtree(local_directory)

        logger.info("End scheduler at %r", addr)

# 需要导入模块: from distributed import Scheduler [as 别名]
# 或者: from distributed.Scheduler import stop [as 别名]
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file):

    if pid_file:
        with open(pid_file, "w") as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)

        atexit.register(del_pid_file)

    if sys.platform.startswith("linux"):
        import resource  # module fails importing on Windows

        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        limit = max(soft, hard // 2)
        resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    given_host = host
    host = host or get_ip()
    if ":" in host and port == 8786:
        host, port = host.rsplit(":", 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    logger.info("-" * 47)

    services = {("http", http_port): HTTPScheduler}
    if _bokeh:
        from distributed.bokeh.scheduler import BokehScheduler

        services[("bokeh", 8788)] = BokehScheduler
    scheduler = Scheduler(ip=ip, loop=loop, services=services)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface

            bokeh_proc = BokehWebInterface(
                host=host,
                http_port=http_port,
                tcp_port=port,
                bokeh_port=bokeh_port,
                bokeh_whitelist=bokeh_whitelist,
                show=show,
                prefix=prefix,
                use_xheaders=use_xheaders,
                quiet=False,
            )
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    logger.info("-" * 47)
    try:
        loop.start()
        loop.close()
    finally:
        scheduler.stop()
        if bokeh_proc:
            bokeh_proc.close()

        logger.info("End scheduler at %s:%d", ip, port)