本文整理汇总了Python中dask.base.tokenize方法的典型用法代码示例。如果您正苦于以下问题:Python base.tokenize方法的具体用法?Python base.tokenize怎么用?Python base.tokenize使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dask.base
的用法示例。
在下文中一共展示了base.tokenize方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: get_tok
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def get_tok(self, source):
"""Get string token from object
Strings are assumed to already be a token; if source or entry, see
if it is a persisted thing ("original_tok" is in its metadata), else
generate its own token.
"""
from dask.base import tokenize
if isinstance(source, str):
return source
if isinstance(source, CatalogEntry):
return source._metadata.get('original_tok', tokenize(source))
if isinstance(source, DataSource):
return source.metadata.get('original_tok', tokenize(source))
raise IndexError
示例2: needs_refresh
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def needs_refresh(self, source):
"""Has the (persisted) source expired in the store
Will return True if the source is not in the store at all, if it's
TTL is set to None, or if more seconds have passed than the TTL.
"""
from dask.base import tokenize
now = time.time()
token = tokenize(source)
if token in self:
s0 = self[token]
if self[token].metadata.get('ttl', None):
then = s0.metadata['timestamp']
if s0.metadata['ttl'] < then - now:
return True
return False
return True
示例3: to_dask
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def to_dask(self, chunks=None, name=None):
"""Return a dask array for this volume.
Parameters
----------
chunks: tuple of ints or tuples of ints
Passed to ``da.from_array``, allows setting the chunks on
initialisation, if the chunking scheme in the stored dataset is not
optimal for the calculations to follow. Note that the chunking should
be compatible with an underlying 4d array.
name: str, optional
An optional keyname for the array. Defaults to hashing the input
Returns
-------
Dask array
"""
import dask.array as da
from dask.base import tokenize
if chunks is None:
chunks = tuple(self.chunk_size) + (self.num_channels, )
if name is None:
name = 'to-dask-' + tokenize(self, chunks)
return da.from_array(self, chunks, name=name)
示例4: _dask_array_vgrid
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def _dask_array_vgrid(self, varname, klevels, k_chunksize):
# return a dask array for a 1D vertical grid var
# single chunk for 1D variables
chunks = ((len(klevels),),)
# manually build dask graph
dsk = {}
token = tokenize(varname, self.store)
name = '-'.join([varname, token])
nz = self.nz if _VAR_METADATA[varname]['dims'] != ['k_p1'] else self.nz+1
task = (_get_1d_chunk, self.store, varname,
list(klevels), nz, self.dtype)
key = name, 0
dsk[key] = task
return dsa.Array(dsk, name, chunks, self.dtype)
示例5: interpolate_xarray
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def interpolate_xarray(xpoints, ypoints, values, shape, kind='cubic',
blocksize=CHUNK_SIZE):
"""Interpolate, generating a dask array."""
vchunks = range(0, shape[0], blocksize)
hchunks = range(0, shape[1], blocksize)
token = tokenize(blocksize, xpoints, ypoints, values, kind, shape)
name = 'interpolate-' + token
from scipy.interpolate import interp2d
interpolator = interp2d(xpoints, ypoints, values, kind=kind)
dskx = {(name, i, j): (interpolate_slice,
slice(vcs, min(vcs + blocksize, shape[0])),
slice(hcs, min(hcs + blocksize, shape[1])),
interpolator)
for i, vcs in enumerate(vchunks)
for j, hcs in enumerate(hchunks)
}
res = da.Array(dskx, name, shape=list(shape),
chunks=(blocksize, blocksize),
dtype=values.dtype)
return DataArray(res, dims=('y', 'x'))
示例6: read_band_blocks
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def read_band_blocks(self, blocksize=CHUNK_SIZE):
"""Read the band in native blocks."""
# For sentinel 1 data, the block are 1 line, and dask seems to choke on that.
band = self.filehandle
shape = band.shape
token = tokenize(blocksize, band)
name = 'read_band-' + token
dskx = dict()
if len(band.block_shapes) != 1:
raise NotImplementedError('Bands with multiple shapes not supported.')
else:
chunks = band.block_shapes[0]
def do_read(the_band, the_window, the_lock):
with the_lock:
return the_band.read(1, None, window=the_window)
for ji, window in band.block_windows(1):
dskx[(name, ) + ji] = (do_read, band, window, self.read_lock)
res = da.Array(dskx, name, shape=list(shape),
chunks=chunks,
dtype=band.dtypes[0])
return DataArray(res, dims=('y', 'x'))
示例7: to_keys
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def to_keys(dsk, *args):
for x in args:
if x is None:
yield None
elif isinstance(x, (da.Array, dd.DataFrame)):
x = delayed(x)
dsk.update(x.dask)
yield x.key
elif isinstance(x, Delayed):
dsk.update(x.dask)
yield x.key
else:
assert not is_dask_collection(x)
key = type(x).__name__ + "-" + tokenize(x)
dsk[key] = x
yield key
示例8: build_refit_graph
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def build_refit_graph(estimator, X, y, best_params, fit_params):
X, y = to_indexable(X, y)
dsk = {}
X_name, y_name = to_keys(dsk, X, y)
fit_params = map_fit_params(dsk, fit_params)
main_token = tokenize(normalize_estimator(estimator), X_name, y_name, fit_params)
best_estimator = "best-estimator-" + main_token
if fit_params:
fit_params = (
dict,
(zip, list(fit_params.keys()), list(pluck(1, fit_params.values()))),
)
dsk[best_estimator] = (
fit_best,
clone(estimator),
best_params,
X_name,
y_name,
fit_params,
)
return dsk, [best_estimator]
示例9: normalize_params
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def normalize_params(params):
"""Take a list of dictionaries, and tokenize/normalize."""
# Collect a set of all fields
fields = set()
for p in params:
fields.update(p)
fields = sorted(fields)
params2 = list(pluck(fields, params, MISSING))
# Non-basic types (including MISSING) are unique to their id
tokens = [
tuple(x if isinstance(x, (int, float, str)) else id(x) for x in p)
for p in params2
]
return fields, tokens, params2
示例10: test_shuffle_split
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_shuffle_split(cls):
assert tokenize(cls(n_splits=3, random_state=0)) == tokenize(
cls(n_splits=3, random_state=0)
)
assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
cls(n_splits=3, random_state=2)
)
assert tokenize(cls(n_splits=3, random_state=0)) != tokenize(
cls(n_splits=4, random_state=0)
)
cv = cls(n_splits=3)
assert compute_n_splits(cv, np_X, np_y, np_groups) == 3
with assert_dask_compute(False):
assert compute_n_splits(cv, da_X, da_y, da_groups) == 3
示例11: test_leave_group_out
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_leave_group_out(cvs):
tokens = []
for cv in cvs:
assert tokenize(cv) == tokenize(cv)
tokens.append(cv)
assert len(set(tokens)) == len(tokens)
cv = cvs[0]
sol = cv.get_n_splits(np_X, np_y, np_groups)
assert compute_n_splits(cv, np_X, np_y, np_groups) == sol
with assert_dask_compute(True):
assert compute_n_splits(cv, da_X, da_y, da_groups) == sol
with assert_dask_compute(False):
assert compute_n_splits(cv, da_X, da_y, np_groups) == sol
示例12: test_old_style_cv
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def test_old_style_cv():
cv1 = _CVIterableWrapper(
[
np.array([True, False, True, False] * 5),
np.array([False, True, False, True] * 5),
]
)
cv2 = _CVIterableWrapper(
[
np.array([True, False, True, False] * 5),
np.array([False, True, True, True] * 5),
]
)
assert tokenize(cv1) == tokenize(cv1)
assert tokenize(cv1) != tokenize(cv2)
sol = cv1.get_n_splits(np_X, np_y, np_groups)
assert compute_n_splits(cv1, np_X, np_y, np_groups) == sol
with assert_dask_compute(False):
assert compute_n_splits(cv1, da_X, da_y, da_groups) == sol
示例13: _tok
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def _tok(self):
import warnings
from dask.base import tokenize
warnings.warn("the _tok attribute is deprecated, please use "
"`dask.base.tokenize(obj)` instead")
return tokenize(self)
示例14: __dask_tokenize__
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def __dask_tokenize__(self):
if self.__tok_cache is None:
from dask.base import tokenize
self.__tok_cache = tokenize(self.__getstate__())
return self.__tok_cache
示例15: __hash__
# 需要导入模块: from dask import base [as 别名]
# 或者: from dask.base import tokenize [as 别名]
def __hash__(self):
from dask.base import tokenize
return int(tokenize(self), 16)