Python xxhash.xxh32方法代码示例

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def lh_perturb(real_dist, g, p):
    n = sum(real_dist)
    noisy_samples = np.zeros(n, dtype=object)
    samples_one = np.random.random_sample(n)
    seeds = np.random.randint(0, n, n)

    counter = 0
    for k, v in enumerate(real_dist):
        for _ in range(v):
            y = x = xxhash.xxh32(str(int(k)), seed=seeds[counter]).intdigest() % g

            if samples_one[counter] > p:
                y = np.random.randint(0, g - 1)
                if y >= x:
                    y += 1
            noisy_samples[counter] = tuple([y, seeds[counter]])
            counter += 1
    return noisy_samples 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def thread_affinity(url, total_worker_count):
	'''
	Ensure only one client ever works on each netloc.
	This maintains better consistency of user-agents
	'''

	# Only limit netlocs if we actually need to.
	if not getModuleForUrl(url).single_thread_fetch(url):
		return True

	netloc = urllib.parse.urlsplit(url).netloc

	m = xxhash.xxh32()
	m.update(netloc.encode("utf-8"))

	nlhash = m.intdigest()
	thread_aff = nlhash % total_worker_count
	# print("Thread affinity:", self.total_worker_count, self.worker_num, thread_aff, self.worker_num == thread_aff)
	return thread_aff 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def perturb():
    global Y
    Y = np.zeros(n)
    for i in range(n):
        v = X[i]
        x = (xxhash.xxh32(str(v), seed=i).intdigest() % g)
        y = x

        p_sample = np.random.random_sample()
        # the following two are equivalent
        # if p_sample > p:
        #     while not y == x:
        #         y = np.random.randint(0, g)
        if p_sample > p - q:
            # perturb
            y = np.random.randint(0, g)
        Y[i] = y 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def reduce_thread_id(thread_id: int) -> str:
    """Make a shorter thread identifier by hashing the original."""
    return xxhash.xxh32(thread_id.to_bytes(8, "little")).hexdigest()[:4] 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def hash_netloc(netloc):

	m = xxhash.xxh32()
	m.update(netloc.encode("utf-8"))
	nlhash = m.intdigest()

	return nlhash 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def hash_file2(fpath, blocksize=65536, hasher='xx64'):
    r"""
    Hashes the data in a file on disk using xxHash

    xxHash is much faster than sha1, bringing computation time down from .57
    seconds to .12 seconds for a 387M file.

    my_weights_fpath_ = ub.truepath('~/tmp/my_weights.pt')


    xdata = 2 ** np.array([8, 12, 14, 16])
    ydatas = ub.ddict(list)
    for blocksize in xdata:
        print('blocksize = {!r}'.format(blocksize))
        ydatas['sha1'].append(ub.Timerit(2).call(ub.hash_file, my_weights_fpath_, hasher='sha1', blocksize=blocksize).min())
        ydatas['sha256'].append(ub.Timerit(2).call(ub.hash_file, my_weights_fpath_, hasher='sha256', blocksize=blocksize).min())
        ydatas['sha512'].append(ub.Timerit(2).call(ub.hash_file, my_weights_fpath_, hasher='sha512', blocksize=blocksize).min())
        ydatas['md5'].append(ub.Timerit(2).call(ub.hash_file, my_weights_fpath_, hasher='md5', blocksize=blocksize).min())
        ydatas['xx32'].append(ub.Timerit(2).call(hash_file2, my_weights_fpath_, hasher='xx32', blocksize=blocksize).min())
        ydatas['xx64'].append(ub.Timerit(2).call(hash_file2, my_weights_fpath_, hasher='xx64', blocksize=blocksize).min())

    import netharn as nh
    nh.util.qtensure()
    nh.util.multi_plot(xdata, ydatas)
    """
    import xxhash
    if hasher == 'xx32':
        hasher = xxhash.xxh32()
    elif hasher == 'xx64':
        hasher = xxhash.xxh64()

    with open(fpath, 'rb') as file:
        buf = file.read(blocksize)
        # otherwise hash the entire file
        while len(buf) > 0:
            hasher.update(buf)
            buf = file.read(blocksize)
    # Get the hashed representation
    text = ub.util_hash._digest_hasher(hasher, hashlen=None,
                                       base=ub.util_hash.DEFAULT_ALPHABET)
    return text 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def _seed(self, val):
        """Returns a unique seed for val and the (optional) namespace."""
        if self._namespace:
            return xxhash.xxh32(
                self._namespace.encode('utf-8') +
                Magnitude.RARE_CHAR +
                val.encode('utf-8')).intdigest()
        else:
            return xxhash.xxh32(val.encode('utf-8')).intdigest() 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def _seed(self, val):
        """Returns a unique seed for val and the (optional) namespace."""
        if self._namespace:
            return xxhash.xxh32(self._namespace + Magnitude.RARE_CHAR +
                                val.encode('utf-8')).intdigest()
        else:
            return xxhash.xxh32(val.encode('utf-8')).intdigest() 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def aggregate():
    global ESTIMATE_DIST
    ESTIMATE_DIST = np.zeros(domain)
    for i in range(n):
        for v in range(domain):
            if Y[i] == (xxhash.xxh32(str(v), seed=i).intdigest() % g):
                ESTIMATE_DIST[v] += 1
    a = 1.0 * g / (p * g - 1)
    b = 1.0 * n / (p * g - 1)
    ESTIMATE_DIST = a * ESTIMATE_DIST - b 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def perturb(self, datas, domain):
        n = len(datas)
        perturbed_datas = np.zeros(n, dtype=object)
        samples_one = np.random.random_sample(n)
        seeds = np.random.randint(0, n, n)
        for i in range(n):
            y = x = xxhash.xxh32(str(int(datas[i])), seed=seeds[i]).intdigest() % self.g
            # y = x = (datas[i] * seeds[i]) % self.g

            if samples_one[i] > self.p:
                y = np.random.randint(0, self.g - 1)
                if y >= x:
                    y += 1
            perturbed_datas[i] = tuple([y, seeds[i]])
        return perturbed_datas 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def support_sr(self, report, value):
        return report[0] == (xxhash.xxh32(str(value), seed=report[1]).intdigest() % self.g)
        # return report[0] == value * report[1] % self.g 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def aggregate(self, domain, perturbed_datas):
        n = len(perturbed_datas)

        ESTIMATE_DIST = np.zeros(domain, dtype=np.int32)
        for i in range(n):
            for v in range(domain):
                x = xxhash.xxh32(str(v), seed=perturbed_datas[i][1]).intdigest() % self.g
                # x = v * perturbed_datas[i][1] % self.g
                if perturbed_datas[i][0] == x:
                    ESTIMATE_DIST[v] += 1

        return ESTIMATE_DIST 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def lh_aggregate(noisy_samples, domain, g, p, q):
    n = len(noisy_samples)

    est = np.zeros(domain, dtype=np.int32)
    for i in range(n):
        for v in range(domain):
            x = xxhash.xxh32(str(v), seed=noisy_samples[i][1]).intdigest() % g
            if noisy_samples[i][0] == x:
                est[v] += 1

    a = 1.0 / (p - q)
    b = n * q / (p - q)
    est = a * est - b

    return est 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def generateLocation1(authticket, lat, lng, alt):
    firstHash = xxhash.xxh32(authticket, seed=static_seed).intdigest()
    locationBytes = d2h(lat) + d2h(lng) + d2h(alt)
    return xxhash.xxh32(locationBytes, seed=firstHash).intdigest() 

# 需要导入模块: import xxhash [as 别名]
# 或者: from xxhash import xxh32 [as 别名]
def generateLocation2(lat, lng, alt):
    locationBytes = d2h(lat) + d2h(lng) + d2h(alt)
    return xxhash.xxh32(locationBytes, seed=static_seed).intdigest()