Python xxhash.xxh32函数代码示例

def hashLocation(authTicket, latitude, longitude, altitude):
    baseHash = xxhash.xxh32(authTicket.SerializeToString(), seed=0x1B845238).intdigest()
    locationBytes = d2h(latitude) + d2h(longitude) + d2h(altitude)

    # Using serialized Auth Ticket
    hashA = xxhash.xxh32(locationBytes, seed=baseHash).intdigest()

    # Hash of location using static seed 0x1B845238
    hashB = xxhash.xxh32(locationBytes, seed=0x1B845238).intdigest()
    return hashA, hashB

def organize_dir(initial_path, db):
    # list of the content of directory
    listdir = [initial_path + "/" + fd for fd in os.listdir(initial_path)]
    # recursivity in all subfolder
    print("+ Recursivity...")
    for i in [d for d in listdir if os.path.isdir(d)]:
        organize_dir(i, db)
    #get media
    print("+ Get media")
    media = [{'path': f, 'date': get_date(f), 'size': os.stat(f).st_size} for f in listdir if
             os.path.isfile(f) and f.__contains__('.') and
             f.split('.')[-1].lower() in IMAGE_EXTENSION]
    #hash and insert in db
    print("+ Insert into db")
    import xxhash
    for m in media:
        print(m['path'])
        #calculate hash
        with open(m['path'], 'rb') as afile:
            digest = str(xxhash.xxh32(afile.read()).hexdigest())
        if db.execute("select * from image where hash = ?", [digest]).fetchall().__len__() > 0:
            print("Already in db")
        db.execute("insert into image (path, year, month, day, hash, size) values (?,?,?,?,?,?) ;",
                   [m['path'], m['date'].year, m['date'].month, m['date'].day, digest, m['size']])
        print(str(m["date"]) + "  " + digest)
    db.commit()

    def _compress_frame(self):
        '''
        frame contains all the blocks, plus frame header and checksum
        '''
        self.dst_file.write(self._frame_header())

        def read_src(buf):
            return self.src_file.readinto(buf)

        self.src_buffer = bytearray(b'\0') * BLOCK_SIZE
        self.dst_buffer = bytearray(
            b'\0') * worst_case_block_length(BLOCK_SIZE)

        xxh = xxhash.xxh32(seed=0)

        nbytes = read_src(self.src_buffer)
        while nbytes != 0:
            block_len = lz4_compress_block(
                self.dst_buffer, memoryview(self.src_buffer)[0:nbytes])
            self.dst_file.write(memoryview(self.dst_buffer)[0:block_len])
            # only pinned buffer, not appropriate here
            xxh.update(bytes(self.src_buffer[0:nbytes]))
            nbytes = read_src(self.src_buffer)

        self.dst_file.write((0).to_bytes(4, 'little'))  # EndMark
        self.dst_file.write(xxh.intdigest().to_bytes(4, 'little'))  # CheckSum

def lz4_encode_old_kafka(payload):
    """Encode payload for 0.8/0.9 brokers -- requires an incorrect header checksum."""
    assert xxhash is not None
    data = lz4_encode(payload)
    header_size = 7
    flg = data[4]
    if not isinstance(flg, int):
        flg = ord(flg)

    content_size_bit = ((flg >> 3) & 1)
    if content_size_bit:
        # Old kafka does not accept the content-size field
        # so we need to discard it and reset the header flag
        flg -= 8
        data = bytearray(data)
        data[4] = flg
        data = bytes(data)
        payload = data[header_size+8:]
    else:
        payload = data[header_size:]

    # This is the incorrect hc
    hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member

    return b''.join([
        data[0:header_size-1],
        hc,
        payload
    ])

def hash_function(shingle, function_id):
    try:
        return xxhash.xxh32(shingle.encode("utf8") * function_id).intdigest()
    except Exception, e:
        print e
        print shingle
        sys.exit(-1)

def decode_lz4_old_kafka(buff):
    """Decode buff for 0.8/0.9 brokers

    Reference impl: https://github.com/dpkp/kafka-python/blob/a00f9ead161e8b05ac953b460950e42fa0e0b7d6/kafka/codec.py#L258
    """
    assert xxhash is not None
    # Kafka's LZ4 code has a bug in its header checksum implementation
    header_size = 7
    if isinstance(buff[4], int):
        flg = buff[4]
    else:
        flg = ord(buff[4])
    content_size_bit = ((flg >> 3) & 1)
    if content_size_bit:
        header_size += 8

    # This should be the correct hc
    hc = xxhash.xxh32(buff[4:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member

    munged_buff = b''.join([
        buff[0:header_size-1],
        hc,
        buff[header_size:]
    ])
    return decode_lz4(munged_buff)

def encode_lz4_old_kafka(buff):
    """Encode buff for 0.8/0.9 brokers -- requires an incorrect header checksum.

    Reference impl: https://github.com/dpkp/kafka-python/blob/a00f9ead161e8b05ac953b460950e42fa0e0b7d6/kafka/codec.py#L227
    """
    assert xxhash is not None
    data = encode_lz4(buff)
    header_size = 7
    flg = data[4]
    if not isinstance(flg, int):
        flg = ord(flg)

    content_size_bit = ((flg >> 3) & 1)
    if content_size_bit:
        # Old kafka does not accept the content-size field
        # so we need to discard it and reset the header flag
        flg -= 8
        data = bytearray(data)
        data[4] = flg
        data = bytes(data)
        buff = data[header_size+8:]
    else:
        buff = data[header_size:]

    # This is the incorrect hc
    hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1]  # pylint: disable-msg=no-member

    return b''.join([
        data[0:header_size-1],
        hc,
        buff
    ])

def minhash_faster_but_less_random(string_set):
    hashers = [xxhash.xxh32(w.encode('utf8')) for w in string_set]
    hashes = np.asarray([h.intdigest() for h in hashers])
    while True:
        hashes *= 2654435761
        hashes %= 2 ** 32
        yield np.min(hashes)

 def _hash(self, item):
     # get Python hash ID of object
     # technique used by Rafa Carrascosa
     # https://github.com/rafacarrascosa/countminsketch
     h = xxhash.xxh32(str(hash(item)))
     for i in range(self.num_rows):
         h.update(str(i))
         yield h.intdigest() % self.num_columns

    def test_XXH32_reset(self):
        x = xxhash.xxh32()
        h = x.intdigest()

        for i in range(10, 50):
            x.update(os.urandom(i))

        x.reset()

        self.assertEqual(h, x.intdigest())

def _hash_with_seed(funcname, seed):
    seed = xxhash.xxh32(seed).intdigest()

    xxh32 = xxhash.xxh32
    spooky32 = spooky.hash32

    if funcname == 'xxhash32':
        return lambda x: xxh32(x, seed=seed).intdigest()
    elif funcname == 'spooky32':
        return lambda x: spooky32(x, seed=seed)
    else:
        raise ValueError('Unknown function name: %s' % funcname)

    def _parse_header(self):
        # IMPORTANT: for simplicity, lz4 configuration is not fully supported
        buf = self.src_file.read(7)
        if len(buf) != 7 or int.from_bytes(buf[0:4], 'little') != MAGIC_NUMBER:
            raise BadFileError

        if buf[4] != int('01100100', 2):  # FLG
            raise BadFileError

        if buf[5] != int('01110000', 2):  # BD
            raise BadFileError

        checksum = xxhash.xxh32(buf[4:6], seed=0).digest()[2]
        if checksum != buf[6]:
            raise BadFileError

    def test_XXH32(self):
        x = xxhash.xxh32()
        x.update('a')
        self.assertEqual(xxhash.xxh32('a').digest(), x.digest())
        x.update('b')
        self.assertEqual(xxhash.xxh32('ab').digest(), x.digest())
        x.update('c')
        self.assertEqual(xxhash.xxh32('abc').digest(), x.digest())

        seed = random.randint(0, 2**32)
        x = xxhash.xxh32(seed=seed)
        x.update('a')
        self.assertEqual(xxhash.xxh32('a', seed).digest(), x.digest())
        x.update('b')
        self.assertEqual(xxhash.xxh32('ab', seed).digest(), x.digest())
        x.update('c')
        self.assertEqual(xxhash.xxh32('abc', seed).digest(), x.digest())

 def _frame_header(self):
     header = bytearray()
     header += MAGIC_NUMBER.to_bytes(4, 'little')
     # default frame descriptor FLG, Version Number 01
     # Block Independenc 1, Block Checksum 0
     # Content Size 0, Content Checksum 1
     FD_FLG = int('01100100', 2)
     # frame descriptor BD
     # Block Max Size 7 -> 4M
     FD_BD = int('01110000', 2)
     # frame descriptor header checksum
     checksum = xxhash.xxh32(bytes([FD_FLG, FD_BD]), seed=0).digest()
     FD_HC = checksum[2]
     header.append(FD_FLG)
     header.append(FD_BD)
     header.append(FD_HC)
     return header

def thread_affinity(url, total_worker_count):
	'''
	Ensure only one client ever works on each netloc.
	This maintains better consistency of user-agents
	'''

	# Only limit netlocs if we actually need to.
	if not getModuleForUrl(url).single_thread_fetch(url):
		return True

	netloc = urllib.parse.urlsplit(url).netloc

	m = xxhash.xxh32()
	m.update(netloc.encode("utf-8"))

	nlhash = m.intdigest()
	thread_aff = nlhash % total_worker_count
	# print("Thread affinity:", self.total_worker_count, self.worker_num, thread_aff, self.worker_num == thread_aff)
	return thread_aff