Python distributed.Client类代码示例

def test_keep_results():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        assert c.get({'x': (inc, 1)}, 'x', keep_results=True) == 2

        assert 'x' in a.data or 'x' in b.data

def main():
    client = Client('localhost:8786')
    A = client.map(set_value, range(100))
    B = client.map(square, A)
    C = client.map(neg, B)
    total = client.submit(sum, C)
    print(progress(total))
    print(total.result())

def test_error():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        assert raises(TypeError,
                lambda: c.get({'x': 1, 'y': (inc, 'x', 'x')}, 'y'))
        assert 'y' not in s.data
        c.close()

def test_get_with_dill():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        dsk = {'x': 1, 'y': (partial(add, 1), 'x')}
        keys = 'y'

        assert c.get(dsk, keys) == 2
        c.close()

def test_get():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        dsk = {'x': 1, 'y': (add, 'x', 'x'), 'z': (inc, 'y')}
        keys = ['y', 'z']

        assert c.get(dsk, keys) == [2, 3]
        c.close()

def test_get_workers():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)
        pid = os.getpid()

        workers = c.get_registered_workers()
        assert set(workers) == set([a.address, b.address])
        assert workers[a.address]['pid'] == pid
        assert workers[b.address]['pid'] == pid

        s.close_workers()
        assert c.get_registered_workers() == {}

def main():
    # Setup logging on the main process:
    _start_logging()

    # Start three worker processes on the local machine:
    client = Client(n_workers=3, threads_per_worker=1)

    # Setup Eliot logging on each worker process:
    client.run(_start_logging)

    # Run the Dask computation in the worker processes:
    result = main_computation()
    print("Result:", result)

def main():
    """."""
    host = os.getenv('DASK_SCHEDULER_HOST', default='localhost')
    port = os.getenv('DASK_SCHEDULER_PORT', default=8786)
    print(host, port)
    client = Client('{}:{}'.format(host, port))
    # client.run(init_logging)
    # client.run_on_scheduler(init_logging)

    # Run some mock functions and gather a result
    data = client.map(print_listdir, range(10))
    future = client.submit(print_values, data)
    progress(future)
    print('')
    result = client.gather(future)
    print(result)

    # Run a second stage which runs some additional processing.
    print('here A')
    data_a = client.map(set_value, range(100))
    print('here B')
    data_b = client.map(square, data_a)
    print('here C')
    data_c = client.map(neg, data_b)
    print('here D')
    # Submit a function application to the scheduler
    total = client.submit(sum, data_c)
    print('here E')
    progress(total)
    print(total.result())
    print('here F')

def cli(scheduler_file, jlab_port, dash_port, notebook_dir,
        hostname, log_level):
    logger = get_logger(log_level)

    logger.info('getting client with scheduler file: %s' % scheduler_file)
    client = Client(scheduler_file=scheduler_file, timeout=30)
    logger.debug('Client: %s' % client)

    logger.debug('Getting hostname where scheduler is running')
    host = client.run_on_scheduler(socket.gethostname)
    logger.info('host is %s' % host)

    logger.info('Starting jupyter lab on host')
    client.run_on_scheduler(start_jlab, host=host, port=jlab_port,
                            notebook_dir=notebook_dir)
    logger.debug('Done.')

    user = os.environ['USER']
    print('Run the following command from your local machine:')
    #print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}@{}'.format(jlab_port, host, jlab_port, dash_port, host, user, hostname))
    print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}'.format(jlab_port, host, jlab_port, dash_port, host,  hostname)) #Modification for existing ssh key
    print('Then open the following URLs:')
    print('\tJupyter lab: http://localhost:{}'.format(jlab_port))
    print('\tDask dashboard: http://localhost:{}'.format(dash_port))

def test_register_collections():
    try:
        import dask.bag as db
    except ImportError:
        return
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        b = db.from_sequence(range(5), npartitions=2).map(inc)
        assert not s.collections
        c.set_collection('mybag', b)
        assert 'mybag' in s.collections

        d = Client(s.address_to_clients)
        b2 = d.get_collection('mybag')

        assert (type(b) == type(b2) and
                b.npartitions == b2.npartitions)
        assert list(b) == list(b2)

        c.close()
        d.close()

def test_multiple_clients():
    with scheduler_and_workers(1) as (s, (a,)):
        c = Client(s.address_to_clients)
        d = Client(s.address_to_clients)

        assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x')

        pool = ThreadPool(2)

        future1 = pool.apply_async(c.get,
                                   args=({'x': 1, 'y': (inc, 'x')}, 'y'))
        future2 = pool.apply_async(d.get,
                                   args=({'a': 1, 'b': (inc, 'a')}, 'b'))

        while not (future1.ready() and future2.ready()):
            sleep(1e-6)

        assert future1.get() == future2.get()
        c.close()
        d.close()

def test_multiple_clients():
    with scheduler_and_workers(1) as (s, (a,)):
        c = Client(s.address_to_clients)
        d = Client(s.address_to_clients)

        assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x')

        def sleep_inc(x):
            sleep(0.5)
            return x + 1

        pool = ThreadPool(2)

        future1 = pool.apply_async(c.get,
                                   args=({'x': 1, 'y': (sleep_inc, 'x')}, 'y'))
        future2 = pool.apply_async(d.get,
                                   args=({'a': 1, 'b': (sleep_inc, 'a')}, 'b'))

        assert future1.get() == future2.get()
        c.close()
        d.close()

    args = parser.parse_args()

    # FIXED parameters
    noise_dim = pd.Index(range(500), name='noise_field')
    mask = xr.open_dataarray('/scratch/pkittiwi/fg1p/hera331_fov_mask.nc')
    if args.xyf_chunks is not None:
        chunks = {'x': args.xyf_chunks[0], 'y': args.xyf_chunks[1],
                  'f': args.xyf_chunks[2]}
    else:
        chunks = None
    # Setup and start Dask Local Cluster
    cluster = LocalCluster(n_workers=args.n_workers, processes=args.processes,
                           threads_per_worker=args.threads_per_worker,
                           scheduler_port=args.scheduler_port,
                           diagnostics_port=args.diagnostics_port)
    client = Client(cluster)
    print('Hostname: {:s}'.format(os.environ['HOSTNAME']))
    print('Dask Scheduler address: {:s}'.format(cluster.scheduler_address))
    print('Dask Dashboard link: {:s}'.format(cluster.dashboard_link))

    # Loop over data parameters and perform calculation
    for bw, fbw, t, s in itertools.product(
            args.bin_width, args.filter_bandwidth, args.theta, args.shift
    ):
        start_time = datetime.now()
        ds = xr.open_mfdataset(
            ['/scratch/pkittiwi/fg1p/binned_noise_map/bin{:.2f}MHz/'
             'fbw{:.2f}MHz/theta{:.1f}/shift{:d}/binned_noise_map_bin{:.2f}MHz_'
             'fbw{:.2f}MHz_theta{:.1f}_shift{:d}_{:03d}.nc'
             .format(bw, fbw, t, s, bw, fbw, t, s, i) for i in range(500)],
            concat_dim=noise_dim, chunks=chunks

def test_status():
    with scheduler_and_workers() as (s, (a, b)):
        c = Client(s.address_to_clients)

        assert c.scheduler_status() == 'OK'
        c.close()

from dask.distributed import Client
import socket

client = Client(scheduler_file='scheduler.json')
print(client)

host = client.run_on_scheduler(socket.gethostname)


def start_jlab(dask_scheduler):
    import subprocess
    proc = subprocess.Popen(['jupyter', 'notebook', '--ip', host])
    dask_scheduler.jlab_proc = proc


client.run_on_scheduler(start_jlab)

print("HOST : %s" % host)