Python base.tokenize方法代码示例

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def get_tok(self, source):
        """Get string token from object

        Strings are assumed to already be a token; if source or entry, see
        if it is a persisted thing ("original_tok" is in its metadata), else
        generate its own token.
        """
        from dask.base import tokenize

        if isinstance(source, str):
            return source

        if isinstance(source, CatalogEntry):
            return source._metadata.get('original_tok', tokenize(source))

        if isinstance(source, DataSource):
            return source.metadata.get('original_tok', tokenize(source))
        raise IndexError 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def needs_refresh(self, source):
        """Has the (persisted) source expired in the store

        Will return True if the source is not in the store at all, if it's
        TTL is set to None, or if more seconds have passed than the TTL.
        """
        from dask.base import tokenize
        now = time.time()
        token = tokenize(source)
        if token in self:
            s0 = self[token]
            if self[token].metadata.get('ttl', None):
                then = s0.metadata['timestamp']
                if s0.metadata['ttl'] < then - now:
                    return True
            return False
        return True 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def to_dask(self, chunks=None, name=None):
    """Return a dask array for this volume.

    Parameters
    ----------
    chunks: tuple of ints or tuples of ints
      Passed to ``da.from_array``, allows setting the chunks on
      initialisation, if the chunking scheme in the stored dataset is not
      optimal for the calculations to follow. Note that the chunking should
      be compatible with an underlying 4d array.
    name: str, optional
      An optional keyname for the array. Defaults to hashing the input

    Returns
    -------
    Dask array
    """
    import dask.array as da
    from dask.base import tokenize

    if chunks is None:
      chunks = tuple(self.chunk_size) + (self.num_channels, )
    if name is None:
      name = 'to-dask-' + tokenize(self, chunks)
    return da.from_array(self, chunks, name=name) 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def _dask_array_vgrid(self, varname, klevels, k_chunksize):
        # return a dask array for a 1D vertical grid var

        # single chunk for 1D variables
        chunks = ((len(klevels),),)

        # manually build dask graph
        dsk = {}
        token = tokenize(varname, self.store)
        name = '-'.join([varname, token])

        nz = self.nz if _VAR_METADATA[varname]['dims'] != ['k_p1'] else self.nz+1
        task = (_get_1d_chunk, self.store, varname,
                list(klevels), nz, self.dtype)

        key = name, 0
        dsk[key] = task

        return dsa.Array(dsk, name, chunks, self.dtype) 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def interpolate_xarray(xpoints, ypoints, values, shape, kind='cubic',
                       blocksize=CHUNK_SIZE):
    """Interpolate, generating a dask array."""
    vchunks = range(0, shape[0], blocksize)
    hchunks = range(0, shape[1], blocksize)

    token = tokenize(blocksize, xpoints, ypoints, values, kind, shape)
    name = 'interpolate-' + token

    from scipy.interpolate import interp2d
    interpolator = interp2d(xpoints, ypoints, values, kind=kind)

    dskx = {(name, i, j): (interpolate_slice,
                           slice(vcs, min(vcs + blocksize, shape[0])),
                           slice(hcs, min(hcs + blocksize, shape[1])),
                           interpolator)
            for i, vcs in enumerate(vchunks)
            for j, hcs in enumerate(hchunks)
            }

    res = da.Array(dskx, name, shape=list(shape),
                   chunks=(blocksize, blocksize),
                   dtype=values.dtype)
    return DataArray(res, dims=('y', 'x')) 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def read_band_blocks(self, blocksize=CHUNK_SIZE):
        """Read the band in native blocks."""
        # For sentinel 1 data, the block are 1 line, and dask seems to choke on that.
        band = self.filehandle

        shape = band.shape
        token = tokenize(blocksize, band)
        name = 'read_band-' + token
        dskx = dict()
        if len(band.block_shapes) != 1:
            raise NotImplementedError('Bands with multiple shapes not supported.')
        else:
            chunks = band.block_shapes[0]

        def do_read(the_band, the_window, the_lock):
            with the_lock:
                return the_band.read(1, None, window=the_window)

        for ji, window in band.block_windows(1):
            dskx[(name, ) + ji] = (do_read, band, window, self.read_lock)

        res = da.Array(dskx, name, shape=list(shape),
                       chunks=chunks,
                       dtype=band.dtypes[0])
        return DataArray(res, dims=('y', 'x')) 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def to_keys(dsk, *args):
    for x in args:
        if x is None:
            yield None
        elif isinstance(x, (da.Array, dd.DataFrame)):
            x = delayed(x)
            dsk.update(x.dask)
            yield x.key
        elif isinstance(x, Delayed):
            dsk.update(x.dask)
            yield x.key
        else:
            assert not is_dask_collection(x)
            key = type(x).__name__ + "-" + tokenize(x)
            dsk[key] = x
            yield key 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def build_refit_graph(estimator, X, y, best_params, fit_params):
    X, y = to_indexable(X, y)
    dsk = {}
    X_name, y_name = to_keys(dsk, X, y)

    fit_params = map_fit_params(dsk, fit_params)
    main_token = tokenize(normalize_estimator(estimator), X_name, y_name, fit_params)

    best_estimator = "best-estimator-" + main_token
    if fit_params:
        fit_params = (
            dict,
            (zip, list(fit_params.keys()), list(pluck(1, fit_params.values()))),
        )
    dsk[best_estimator] = (
        fit_best,
        clone(estimator),
        best_params,
        X_name,
        y_name,
        fit_params,
    )
    return dsk, [best_estimator] 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def normalize_params(params):
    """Take a list of dictionaries, and tokenize/normalize."""
    # Collect a set of all fields
    fields = set()
    for p in params:
        fields.update(p)
    fields = sorted(fields)

    params2 = list(pluck(fields, params, MISSING))
    # Non-basic types (including MISSING) are unique to their id
    tokens = [
        tuple(x if isinstance(x, (int, float, str)) else id(x) for x in p)
        for p in params2
    ]

    return fields, tokens, params2 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_shuffle_split(cls):
    assert tokenize(cls(n_splits=3, random_state=0)) == tokenize(
        cls(n_splits=3, random_state=0)
    )

    assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
        cls(n_splits=3, random_state=2)
    )

    assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
        cls(n_splits=4, random_state=0)
    )

    cv = cls(n_splits=3)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == 3

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == 3 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_leave_group_out(cvs):
    tokens = []
    for cv in cvs:
        assert tokenize(cv) == tokenize(cv)
        tokens.append(cv)
    assert len(set(tokens)) == len(tokens)

    cv = cvs[0]
    sol = cv.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv, np_X, np_y, np_groups) == sol

    with assert_dask_compute(True):
        assert compute_n_splits(cv, da_X, da_y, da_groups) == sol

    with assert_dask_compute(False):
        assert compute_n_splits(cv, da_X, da_y, np_groups) == sol 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_old_style_cv():
    cv1 = _CVIterableWrapper(
        [
            np.array([True, False, True, False] * 5),
            np.array([False, True, False, True] * 5),
        ]
    )
    cv2 = _CVIterableWrapper(
        [
            np.array([True, False, True, False] * 5),
            np.array([False, True, True, True] * 5),
        ]
    )
    assert tokenize(cv1) == tokenize(cv1)
    assert tokenize(cv1) != tokenize(cv2)

    sol = cv1.get_n_splits(np_X, np_y, np_groups)
    assert compute_n_splits(cv1, np_X, np_y, np_groups) == sol
    with assert_dask_compute(False):
        assert compute_n_splits(cv1, da_X, da_y, da_groups) == sol 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def _tok(self):
        import warnings
        from dask.base import tokenize
        warnings.warn("the _tok attribute is deprecated, please use "
                      "`dask.base.tokenize(obj)` instead")
        return tokenize(self) 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def __dask_tokenize__(self):
        if self.__tok_cache is None:
            from dask.base import tokenize
            self.__tok_cache = tokenize(self.__getstate__())
        return self.__tok_cache 

# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def __hash__(self):
        from dask.base import tokenize
        return int(tokenize(self), 16)