Python distributed.Client方法代码示例

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def main(args):
    opts, args = parseArgs(args)
    inDF = pd.read_csv(opts.inFile, sep = '\t', index_col = 0, header = 0)

    client = Client(processes = False)    

    if opts.algo == 'GENIE3':
        network = genie3(inDF, client_or_address = client)
        network.to_csv(opts.outFile, index = False, sep = '\t')

    elif opts.algo == 'GRNBoost2':
        network = grnboost2(inDF, client_or_address = client)
        network.to_csv(opts.outFile, index = False, sep = '\t')

    else:
        print("Wrong algorithm name. Should either be GENIE3 or GRNBoost2.") 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def spawn_cluster_and_client(
    address: Optional[str] = None, **kwargs
) -> Tuple[Optional[LocalCluster], Optional[Client]]:
    """
    If provided an address, create a Dask Client connection.
    If not provided an address, create a LocalCluster and Client connection.
    If not provided an address, other Dask kwargs are accepted and passed down to the
    LocalCluster object.

    Notes
    -----
    When using this function, the processing machine or container must have networking
    capabilities enabled to function properly.
    """
    cluster = None
    if address is not None:
        client = Client(address)
        log.info(f"Connected to Remote Dask Cluster: {client}")
    else:
        cluster = LocalCluster(**kwargs)
        client = Client(cluster)
        log.info(f"Connected to Local Dask Cluster: {client}")

    return cluster, client 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def shutdown_cluster_and_client(
    cluster: Optional[LocalCluster], client: Optional[Client]
) -> Tuple[Optional[LocalCluster], Optional[Client]]:
    """
    Shutdown a cluster and client.

    Notes
    -----
    When using this function, the processing machine or container must have networking
    capabilities enabled to function properly.
    """
    if cluster is not None:
        cluster.close()
    if client is not None:
        client.shutdown()
        client.close()

    return cluster, client 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def test_scheduler_proxy(proxy, cluster_and_security):
    cluster, security = cluster_and_security

    proxied_addr = f"gateway://{proxy.tcp_address}/temp"

    # Add a route
    await proxy.add_route(kind="SNI", sni="temp", target=cluster.scheduler_address)

    # Proxy works
    async def test_works():
        async with Client(proxied_addr, security=security, asynchronous=True) as client:
            res = await client.run_on_scheduler(lambda x: x + 1, 1)
            assert res == 2

    await with_retries(test_works, 5)

    # Remove the route
    await proxy.remove_route(kind="SNI", sni="temp")
    await proxy.remove_route(kind="SNI", sni="temp") 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def get_client(self, set_as_default=True):
        """Get a ``Client`` for this cluster.

        Returns
        -------
        client : dask.distributed.Client
        """
        client = Client(
            self,
            security=self.security,
            set_as_default=set_as_default,
            asynchronous=self.asynchronous,
            loop=self.loop,
        )
        if not self.asynchronous:
            self._clients.add(client)
        return client 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def __init__(self, dask_client=None, client_max_jobs=np.inf,
                 default_pickle=False, batch_size=1):
        super().__init__()

        # Assign Client
        if dask_client is None:
            dask_client = Client()
        self.my_client = dask_client

        # Client options
        self.client_max_jobs = client_max_jobs

        # Job state
        self.jobs_queued = 0

        # For dask, we use cloudpickle by default
        self.default_pickle = default_pickle

        # Batchsize
        self.batch_size = batch_size 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def start_cluster(diagnostics_port=0):
    "Set up a LocalCluster for distributed"
    
    hostname = socket.gethostname()
    n_workers = os.cpu_count() // 2
    cluster = LocalCluster(ip='localhost',
                       n_workers=n_workers,
                       diagnostics_port=diagnostics_port,
                       memory_limit=6e9)
    client = Client(cluster)

    params = { 'bokeh_port': cluster.scheduler.services['bokeh'].port,
           'user': getpass.getuser(),
           'scheduler_ip': cluster.scheduler.ip,
           'hostname': hostname, }

    print("If the link to the dashboard below doesn't work, run this command on a local terminal to set up a SSH tunnel:")
    print()
    print("  ssh -N -L {bokeh_port}:{scheduler_ip}:{bokeh_port} {hostname}.nci.org.au -l {user}".format(**params) )
    
    return client 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def _exec_calcs(calcs, parallelize=False, client=None, **compute_kwargs):
    """Execute the given calculations.

    Parameters
    ----------
    calcs : Sequence of ``aospy.Calc`` objects
    parallelize : bool, default False
        Whether to submit the calculations in parallel or not
    client : distributed.Client or None
        The distributed Client used if parallelize is set to True; if None
        a distributed LocalCluster is used.
    compute_kwargs : dict of keyword arguments passed to ``Calc.compute``

    Returns
    -------
    A list of the values returned by each Calc object that was executed.
    """
    if parallelize:
        def func(calc):
            """Wrap _compute_or_skip_on_error to require only the calc
            argument"""
            if 'write_to_tar' in compute_kwargs:
                compute_kwargs['write_to_tar'] = False
            return _compute_or_skip_on_error(calc, compute_kwargs)

        if client is None:
            n_workers = _n_workers_for_local_cluster(calcs)
            with distributed.LocalCluster(n_workers=n_workers) as cluster:
                with distributed.Client(cluster) as client:
                    result = _submit_calcs_on_client(calcs, client, func)
        else:
            result = _submit_calcs_on_client(calcs, client, func)
        if compute_kwargs['write_to_tar']:
            _serial_write_to_tar(calcs)
        return result
    else:
        return [_compute_or_skip_on_error(calc, compute_kwargs)
                for calc in calcs] 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def external_client():
    # Explicitly specify we want only 4 workers so that when running on
    # continuous integration we don't request too many.
    cluster = distributed.LocalCluster(n_workers=4)
    client = distributed.Client(cluster)
    yield client
    client.close()
    cluster.close() 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def __init__(self, cluster_address=None):
        super().__init__(parallelism=0)
        if cluster_address is None:
            cluster_address = conf.get('dask', 'cluster_address')
        if not cluster_address:
            raise ValueError('Please provide a Dask cluster address in airflow.cfg')
        self.cluster_address = cluster_address
        # ssl / tls parameters
        self.tls_ca = conf.get('dask', 'tls_ca')
        self.tls_key = conf.get('dask', 'tls_key')
        self.tls_cert = conf.get('dask', 'tls_cert')
        self.client: Optional[Client] = None
        self.futures: Optional[Dict[Future, TaskInstanceKeyType]] = None 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def start(self) -> None:
        if self.tls_ca or self.tls_key or self.tls_cert:
            security = Security(
                tls_client_key=self.tls_key,
                tls_client_cert=self.tls_cert,
                tls_ca_file=self.tls_ca,
                require_encryption=True,
            )
        else:
            security = None

        self.client = Client(self.cluster_address, security=security)
        self.futures = {} 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def cluster_and_client(address: Optional[str] = None, **kwargs):
    """
    If provided an address, create a Dask Client connection.
    If not provided an address, create a LocalCluster and Client connection.
    If not provided an address, other Dask kwargs are accepted and passed down to the
    LocalCluster object.

    These objects will only live for the duration of this context manager.

    Examples
    --------
    >>> with cluster_and_client() as (cluster, client):
    ...     img1 = AICSImage("1.tiff")
    ...     img2 = AICSImage("2.czi")
    ...     other processing

    Notes
    -----
    When using this context manager, the processing machine or container must have
    networking capabilities enabled to function properly.
    """
    try:
        cluster, client = spawn_cluster_and_client(address=address, **kwargs)
        yield cluster, client
    finally:
        shutdown_cluster_and_client(cluster=cluster, client=client) 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def cli_option_scheduler(func):
    """Decorator for adding a pre-defined, reusable CLI option `--scheduler`."""

    # noinspection PyUnusedLocal
    def _callback(ctx: click.Context, param: click.Option, value: Optional[str]):
        if not value:
            return

        address_and_kwargs = value.split("?", 2)
        if len(address_and_kwargs) == 2:
            address, kwargs_string = address_and_kwargs
            kwargs = parse_cli_kwargs(kwargs_string, metavar="SCHEDULER")
        else:
            address, = address_and_kwargs
            kwargs = dict()

        try:
            # The Dask Client registers itself as the default Dask scheduler, and so runs dask.array used by xarray
            import distributed
            scheduler_client = distributed.Client(address, **kwargs)
            ctx_obj = ctx.ensure_object(dict)
            if ctx_obj is not None:
                ctx_obj["scheduler"] = scheduler_client
            return scheduler_client
        except ValueError as e:
            raise click.BadParameter(f'Failed to create Dask scheduler client: {e}') from e

    return click.option(
        '--scheduler',
        metavar='SCHEDULER',
        help="Enable distributed computing using the Dask scheduler identified by SCHEDULER. "
             "SCHEDULER can have the form <address>?<keyword>=<value>,... where <address> "
             "is <host> or <host>:<port> and specifies the scheduler's address in your network. "
             "For more information on distributed computing "
             "using Dask, refer to http://distributed.dask.org/. "
             "Pairs of <keyword>=<value> are passed to the Dask client. "
             "Refer to http://distributed.dask.org/en/latest/api.html#distributed.Client",
        callback=_callback)(func) 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def setup():
    from distributed import LocalCluster, Client
    cluster = LocalCluster(n_workers=1, threads_per_worker=1, processes=False)
    use_distributed(Client(cluster)) 

# 需要导入模块: import distributed [as 别名]
# 或者: from distributed import Client [as 别名]
def test_basic(loop, nanny, mpirun):
    with tmpfile(extension="json") as fn:

        cmd = mpirun + ["-np", "4", "dask-mpi", "--scheduler-file", fn, nanny]

        with popen(cmd):
            with Client(scheduler_file=fn) as c:
                start = time()
                while len(c.scheduler_info()["workers"]) != 3:
                    assert time() < start + 10
                    sleep(0.2)

                assert c.submit(lambda x: x + 1, 10, workers=1).result() == 11