本文整理汇总了Python中xxhash.xxh32函数的典型用法代码示例。如果您正苦于以下问题:Python xxh32函数的具体用法?Python xxh32怎么用?Python xxh32使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了xxh32函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: hashLocation
def hashLocation(authTicket, latitude, longitude, altitude):
baseHash = xxhash.xxh32(authTicket.SerializeToString(), seed=0x1B845238).intdigest()
locationBytes = d2h(latitude) + d2h(longitude) + d2h(altitude)
# Using serialized Auth Ticket
hashA = xxhash.xxh32(locationBytes, seed=baseHash).intdigest()
# Hash of location using static seed 0x1B845238
hashB = xxhash.xxh32(locationBytes, seed=0x1B845238).intdigest()
return hashA, hashB
示例2: organize_dir
def organize_dir(initial_path, db):
# list of the content of directory
listdir = [initial_path + "/" + fd for fd in os.listdir(initial_path)]
# recursivity in all subfolder
print("+ Recursivity...")
for i in [d for d in listdir if os.path.isdir(d)]:
organize_dir(i, db)
#get media
print("+ Get media")
media = [{'path': f, 'date': get_date(f), 'size': os.stat(f).st_size} for f in listdir if
os.path.isfile(f) and f.__contains__('.') and
f.split('.')[-1].lower() in IMAGE_EXTENSION]
#hash and insert in db
print("+ Insert into db")
import xxhash
for m in media:
print(m['path'])
#calculate hash
with open(m['path'], 'rb') as afile:
digest = str(xxhash.xxh32(afile.read()).hexdigest())
if db.execute("select * from image where hash = ?", [digest]).fetchall().__len__() > 0:
print("Already in db")
db.execute("insert into image (path, year, month, day, hash, size) values (?,?,?,?,?,?) ;",
[m['path'], m['date'].year, m['date'].month, m['date'].day, digest, m['size']])
print(str(m["date"]) + " " + digest)
db.commit()
示例3: _compress_frame
def _compress_frame(self):
'''
frame contains all the blocks, plus frame header and checksum
'''
self.dst_file.write(self._frame_header())
def read_src(buf):
return self.src_file.readinto(buf)
self.src_buffer = bytearray(b'\0') * BLOCK_SIZE
self.dst_buffer = bytearray(
b'\0') * worst_case_block_length(BLOCK_SIZE)
xxh = xxhash.xxh32(seed=0)
nbytes = read_src(self.src_buffer)
while nbytes != 0:
block_len = lz4_compress_block(
self.dst_buffer, memoryview(self.src_buffer)[0:nbytes])
self.dst_file.write(memoryview(self.dst_buffer)[0:block_len])
# only pinned buffer, not appropriate here
xxh.update(bytes(self.src_buffer[0:nbytes]))
nbytes = read_src(self.src_buffer)
self.dst_file.write((0).to_bytes(4, 'little')) # EndMark
self.dst_file.write(xxh.intdigest().to_bytes(4, 'little')) # CheckSum
示例4: lz4_encode_old_kafka
def lz4_encode_old_kafka(payload):
"""Encode payload for 0.8/0.9 brokers -- requires an incorrect header checksum."""
assert xxhash is not None
data = lz4_encode(payload)
header_size = 7
flg = data[4]
if not isinstance(flg, int):
flg = ord(flg)
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
# Old kafka does not accept the content-size field
# so we need to discard it and reset the header flag
flg -= 8
data = bytearray(data)
data[4] = flg
data = bytes(data)
payload = data[header_size+8:]
else:
payload = data[header_size:]
# This is the incorrect hc
hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member
return b''.join([
data[0:header_size-1],
hc,
payload
])
示例5: hash_function
def hash_function(shingle, function_id):
try:
return xxhash.xxh32(shingle.encode("utf8") * function_id).intdigest()
except Exception, e:
print e
print shingle
sys.exit(-1)
示例6: decode_lz4_old_kafka
def decode_lz4_old_kafka(buff):
"""Decode buff for 0.8/0.9 brokers
Reference impl: https://github.com/dpkp/kafka-python/blob/a00f9ead161e8b05ac953b460950e42fa0e0b7d6/kafka/codec.py#L258
"""
assert xxhash is not None
# Kafka's LZ4 code has a bug in its header checksum implementation
header_size = 7
if isinstance(buff[4], int):
flg = buff[4]
else:
flg = ord(buff[4])
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
header_size += 8
# This should be the correct hc
hc = xxhash.xxh32(buff[4:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member
munged_buff = b''.join([
buff[0:header_size-1],
hc,
buff[header_size:]
])
return decode_lz4(munged_buff)
示例7: encode_lz4_old_kafka
def encode_lz4_old_kafka(buff):
"""Encode buff for 0.8/0.9 brokers -- requires an incorrect header checksum.
Reference impl: https://github.com/dpkp/kafka-python/blob/a00f9ead161e8b05ac953b460950e42fa0e0b7d6/kafka/codec.py#L227
"""
assert xxhash is not None
data = encode_lz4(buff)
header_size = 7
flg = data[4]
if not isinstance(flg, int):
flg = ord(flg)
content_size_bit = ((flg >> 3) & 1)
if content_size_bit:
# Old kafka does not accept the content-size field
# so we need to discard it and reset the header flag
flg -= 8
data = bytearray(data)
data[4] = flg
data = bytes(data)
buff = data[header_size+8:]
else:
buff = data[header_size:]
# This is the incorrect hc
hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member
return b''.join([
data[0:header_size-1],
hc,
buff
])
示例8: minhash_faster_but_less_random
def minhash_faster_but_less_random(string_set):
hashers = [xxhash.xxh32(w.encode('utf8')) for w in string_set]
hashes = np.asarray([h.intdigest() for h in hashers])
while True:
hashes *= 2654435761
hashes %= 2 ** 32
yield np.min(hashes)
示例9: _hash
def _hash(self, item):
# get Python hash ID of object
# technique used by Rafa Carrascosa
# https://github.com/rafacarrascosa/countminsketch
h = xxhash.xxh32(str(hash(item)))
for i in range(self.num_rows):
h.update(str(i))
yield h.intdigest() % self.num_columns
示例10: test_XXH32_reset
def test_XXH32_reset(self):
x = xxhash.xxh32()
h = x.intdigest()
for i in range(10, 50):
x.update(os.urandom(i))
x.reset()
self.assertEqual(h, x.intdigest())
示例11: _hash_with_seed
def _hash_with_seed(funcname, seed):
seed = xxhash.xxh32(seed).intdigest()
xxh32 = xxhash.xxh32
spooky32 = spooky.hash32
if funcname == 'xxhash32':
return lambda x: xxh32(x, seed=seed).intdigest()
elif funcname == 'spooky32':
return lambda x: spooky32(x, seed=seed)
else:
raise ValueError('Unknown function name: %s' % funcname)
示例12: _parse_header
def _parse_header(self):
# IMPORTANT: for simplicity, lz4 configuration is not fully supported
buf = self.src_file.read(7)
if len(buf) != 7 or int.from_bytes(buf[0:4], 'little') != MAGIC_NUMBER:
raise BadFileError
if buf[4] != int('01100100', 2): # FLG
raise BadFileError
if buf[5] != int('01110000', 2): # BD
raise BadFileError
checksum = xxhash.xxh32(buf[4:6], seed=0).digest()[2]
if checksum != buf[6]:
raise BadFileError
示例13: test_XXH32
def test_XXH32(self):
x = xxhash.xxh32()
x.update('a')
self.assertEqual(xxhash.xxh32('a').digest(), x.digest())
x.update('b')
self.assertEqual(xxhash.xxh32('ab').digest(), x.digest())
x.update('c')
self.assertEqual(xxhash.xxh32('abc').digest(), x.digest())
seed = random.randint(0, 2**32)
x = xxhash.xxh32(seed=seed)
x.update('a')
self.assertEqual(xxhash.xxh32('a', seed).digest(), x.digest())
x.update('b')
self.assertEqual(xxhash.xxh32('ab', seed).digest(), x.digest())
x.update('c')
self.assertEqual(xxhash.xxh32('abc', seed).digest(), x.digest())
示例14: _frame_header
def _frame_header(self):
header = bytearray()
header += MAGIC_NUMBER.to_bytes(4, 'little')
# default frame descriptor FLG, Version Number 01
# Block Independenc 1, Block Checksum 0
# Content Size 0, Content Checksum 1
FD_FLG = int('01100100', 2)
# frame descriptor BD
# Block Max Size 7 -> 4M
FD_BD = int('01110000', 2)
# frame descriptor header checksum
checksum = xxhash.xxh32(bytes([FD_FLG, FD_BD]), seed=0).digest()
FD_HC = checksum[2]
header.append(FD_FLG)
header.append(FD_BD)
header.append(FD_HC)
return header
示例15: thread_affinity
def thread_affinity(url, total_worker_count):
'''
Ensure only one client ever works on each netloc.
This maintains better consistency of user-agents
'''
# Only limit netlocs if we actually need to.
if not getModuleForUrl(url).single_thread_fetch(url):
return True
netloc = urllib.parse.urlsplit(url).netloc
m = xxhash.xxh32()
m.update(netloc.encode("utf-8"))
nlhash = m.intdigest()
thread_aff = nlhash % total_worker_count
# print("Thread affinity:", self.total_worker_count, self.worker_num, thread_aff, self.worker_num == thread_aff)
return thread_aff