Python joblib.hash方法代码示例

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_setting_ndarray(adata):
    adata.obsp["a"] = np.ones((M, M))
    adata.varp["a"] = np.ones((N, N))
    assert np.all(adata.obsp["a"] == np.ones((M, M)))
    assert np.all(adata.varp["a"] == np.ones((N, N)))

    h = joblib.hash(adata)
    with pytest.raises(ValueError):
        adata.obsp["b"] = np.ones((int(M / 2), M))
    with pytest.raises(ValueError):
        adata.obsp["b"] = np.ones((M, int(M * 2)))
    with pytest.raises(ValueError):
        adata.varp["b"] = np.ones((int(N / 2), 10))
    with pytest.raises(ValueError):
        adata.varp["b"] = np.ones((N, int(N * 2)))
    assert h == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_setting_sparse(adata):
    obsp_sparse = sparse.random(M, M)
    adata.obsp["a"] = obsp_sparse
    assert not np.any((adata.obsp["a"] != obsp_sparse).data)

    varp_sparse = sparse.random(N, N)
    adata.varp["a"] = varp_sparse
    assert not np.any((adata.varp["a"] != varp_sparse).data)

    h = joblib.hash(adata)

    bad_obsp_sparse = sparse.random(M * 2, M)
    with pytest.raises(ValueError):
        adata.obsp["b"] = bad_obsp_sparse

    bad_varp_sparse = sparse.random(N * 2, N)
    with pytest.raises(ValueError):
        adata.varp["b"] = bad_varp_sparse

    assert h == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_setting_ndarray(adata):
    adata.obsm["a"] = np.ones((M, 10))
    adata.varm["a"] = np.ones((N, 10))
    assert np.all(adata.obsm["a"] == np.ones((M, 10)))
    assert np.all(adata.varm["a"] == np.ones((N, 10)))

    h = joblib.hash(adata)
    with pytest.raises(ValueError):
        adata.obsm["b"] = np.ones((int(M / 2), 10))
    with pytest.raises(ValueError):
        adata.obsm["b"] = np.ones((int(M * 2), 10))
    with pytest.raises(ValueError):
        adata.varm["b"] = np.ones((int(N / 2), 10))
    with pytest.raises(ValueError):
        adata.varm["b"] = np.ones((int(N * 2), 10))
    assert h == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_setting_sparse(adata):
    obsm_sparse = sparse.random(M, 100)
    adata.obsm["a"] = obsm_sparse
    assert not np.any((adata.obsm["a"] != obsm_sparse).data)

    varm_sparse = sparse.random(N, 100)
    adata.varm["a"] = varm_sparse
    assert not np.any((adata.varm["a"] != varm_sparse).data)

    h = joblib.hash(adata)

    bad_obsm_sparse = sparse.random(M * 2, M)
    with pytest.raises(ValueError):
        adata.obsm["b"] = bad_obsm_sparse

    bad_varm_sparse = sparse.random(N * 2, N)
    with pytest.raises(ValueError):
        adata.varm["b"] = bad_varm_sparse

    assert h == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_var(adata, subset_func):
    init_hash = joblib.hash(adata)

    subset = adata[:, subset_func(adata.var_names)]

    new_var = pd.DataFrame(
        dict(a=np.ones(subset.n_vars), b=np.ones(subset.n_vars)),
        index=subset.var_names,
    )

    assert subset.is_view
    subset.var = new_var
    assert not subset.is_view
    assert np.all(subset.var == new_var)

    assert joblib.hash(adata) == init_hash 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_obsm(adata):
    init_hash = joblib.hash(adata)

    dim0_size = np.random.randint(2, adata.shape[0] - 1)
    dim1_size = np.random.randint(1, 99)
    orig_obsm_val = adata.obsm["o"].copy()
    subset_idx = np.random.choice(adata.obs_names, dim0_size, replace=False)

    subset = adata[subset_idx, :]
    assert subset.is_view
    subset.obsm = dict(o=np.ones((dim0_size, dim1_size)))
    assert not subset.is_view
    assert np.all(orig_obsm_val == adata.obsm["o"])  # Checking for mutation
    assert np.all(subset.obsm["o"] == np.ones((dim0_size, dim1_size)))

    subset = adata[subset_idx, :]
    subset_hash = joblib.hash(subset)
    with pytest.raises(ValueError):
        subset.obsm = dict(o=np.ones((dim0_size + 1, dim1_size)))
    with pytest.raises(ValueError):
        subset.varm = dict(o=np.ones((dim0_size - 1, dim1_size)))
    assert subset_hash == joblib.hash(subset)

    # Only modification have been made to a view
    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_not_set_subset_X(matrix_type, subset_func):
    adata = ad.AnnData(matrix_type(asarray(sparse.random(20, 20))))
    init_hash = joblib.hash(adata)
    orig_X_val = adata.X.copy()
    while True:
        subset_idx = slice_subset(adata.obs_names)
        if len(adata[subset_idx, :]) > 2:
            break
    subset = adata[subset_idx, :]

    subset = adata[:, subset_idx]

    internal_idx = _normalize_index(
        subset_func(np.arange(subset.X.shape[1])), subset.var_names
    )
    assert subset.is_view
    subset.X[:, internal_idx] = 1
    assert not subset.is_view
    assert not np.any(asarray(adata.X != orig_X_val))

    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_subset_obsm(adata, subset_func):
    init_hash = joblib.hash(adata)
    orig_obsm_val = adata.obsm["o"].copy()

    while True:
        subset_idx = slice_subset(adata.obs_names)
        if len(adata[subset_idx, :]) > 2:
            break
    subset = adata[subset_idx, :]

    internal_idx = _normalize_index(
        subset_func(np.arange(subset.obsm["o"].shape[0])), subset.obs_names
    )

    assert subset.is_view
    subset.obsm["o"][internal_idx] = 1
    assert not subset.is_view
    assert np.all(adata.obsm["o"] == orig_obsm_val)

    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_subset_varm(adata, subset_func):
    init_hash = joblib.hash(adata)
    orig_varm_val = adata.varm["o"].copy()

    while True:
        subset_idx = slice_subset(adata.var_names)
        if (adata[:, subset_idx]).shape[1] > 2:
            break
    subset = adata[:, subset_idx]

    internal_idx = _normalize_index(
        subset_func(np.arange(subset.varm["o"].shape[0])), subset.var_names
    )

    assert subset.is_view
    subset.varm["o"][internal_idx] = 1
    assert not subset.is_view
    assert np.all(adata.varm["o"] == orig_varm_val)

    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_view_delitem(attr):
    adata = gen_adata((10, 10))
    getattr(adata, attr)["to_delete"] = np.ones((10, 10))
    # Shouldn’t be a subclass, should be an ndarray
    assert type(getattr(adata, attr)["to_delete"]) is np.ndarray
    view = adata[5:7, :][:, :5]
    adata_hash = joblib.hash(adata)
    view_hash = joblib.hash(view)

    getattr(view, attr).__delitem__("to_delete")

    assert not view.is_view
    assert "to_delete" not in getattr(view, attr)
    assert "to_delete" in getattr(adata, attr)
    assert adata_hash == joblib.hash(adata)
    assert view_hash != joblib.hash(view) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def get_data_hashes(self, exclude_list=None, hash_type='sha1'):
        """Compute a the hash of data items

        exclude_list: list or None
            List of attributes to skip.
            if None, skips ['metadata']

        hash_type: {'sha1', 'md5', 'sha256'}
            Algorithm to use for hashing
        """
        if exclude_list is None:
            exclude_list = ['metadata']

        ret = {'hash_type': hash_type}
        for key, value in self.items():
            if key in exclude_list:
                continue
            ret[f"{key}_hash"] = joblib.hash(value, hash_name=hash_type)
        return ret 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def train_model(algorithm_params=None,
                run_number=0, *, dataset_name, algorithm_name, hash_type,
                **kwargs):
    """Train a model using the specified algorithm using the given dataset.

    """
    metadata = {}
    ds = Dataset.load(dataset_name)
    metadata['data_hash'] = joblib.hash(ds.data, hash_name=hash_type)
    metadata['target_hash'] = joblib.hash(ds.target, hash_name=hash_type)
    model = available_algorithms(keys_only=False)[algorithm_name]
    model.set_params(**algorithm_params)
    start_time = time.time()
    model.fit(ds.data, y=ds.target)
    end_time = record_time_interval('train_model', start_time)
    metadata['start_time'] = start_time
    metadata['duration'] = end_time - start_time
    return model, metadata 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_obsm_key(adata):
    init_hash = joblib.hash(adata)

    orig_obsm_val = adata.obsm["o"].copy()
    subset_obsm = adata[:50]
    assert subset_obsm.is_view
    subset_obsm.obsm["o"] = np.ones((50, 20))
    assert not subset_obsm.is_view
    assert np.all(adata.obsm["o"] == orig_obsm_val)

    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_varm_key(adata):
    init_hash = joblib.hash(adata)

    orig_varm_val = adata.varm["o"].copy()
    subset_varm = adata[:, :50]
    assert subset_varm.is_view
    subset_varm.varm["o"] = np.ones((50, 20))
    assert not subset_varm.is_view
    assert np.all(adata.varm["o"] == orig_varm_val)

    assert init_hash == joblib.hash(adata) 

# 需要导入模块: import joblib [as 别名]
# 或者: from joblib import hash [as 别名]
def test_set_obs(adata, subset_func):
    init_hash = joblib.hash(adata)

    subset = adata[subset_func(adata.obs_names), :]

    new_obs = pd.DataFrame(
        dict(a=np.ones(subset.n_obs), b=np.ones(subset.n_obs)), index=subset.obs_names,
    )

    assert subset.is_view
    subset.obs = new_obs
    assert not subset.is_view
    assert np.all(subset.obs == new_obs)

    assert joblib.hash(adata) == init_hash