Python xxhash.xxh64函数代码示例

    def __init__(self, tokens, length=100000):

        """Calculates a Charikar simhash with appropriate bitlength.
        
        Input can be any iterable, but for strings it will automatically
        break it into words first, assuming you don't want to iterate
        over the individual characters. Returns nothing.
        
        """
        if isinstance(tokens,basestring):
            tokens = tokens.split()

        v = {}
        if isinstance(tokens,dict):
            for value,w in tokens.iteritems():
                k = xxhash.xxh64(value).intdigest()
                x = v.get(k%length,0)
                if k & 1 << 63:
                    v[k%length] = x + w
                else:
                    v[k%length] = x - w
        else:
            for value in tokens:
                k = xxhash.xxh64(value).intdigest()
                x = v.get(k%length,0)
                if k & 1 << 63:
                    v[k%length] = x + 1
                else:
                    v[k%length] = x - 1
    
        self.hash = v
        self.vector = v

    def _xxhash(self):
        """
        An xxhash.b64 hash of the array.

        Returns
        -------------
        xx: int, xxhash.xxh64 hash of array.
        """
        # repeat the bookkeeping to get a contiguous array inside
        # the function to avoid additional function calls
        # these functions are called millions of times so everything helps
        if self._modified_x or not hasattr(self, '_hashed_xx'):
            if self.flags['C_CONTIGUOUS']:
                hasher = xxhash.xxh64(self)
                self._hashed_xx = hasher.intdigest()
            else:
                # the case where we have sliced our nice
                # contiguous array into a non- contiguous block
                # for example (note slice *after* track operation):
                # t = util.tracked_array(np.random.random(10))[::-1]
                contiguous = np.ascontiguousarray(self)
                hasher = xxhash.xxh64(contiguous)
                self._hashed_xx = hasher.intdigest()
        self._modified_x = False
        return self._hashed_xx

 def __hash_from_argument(self, argument):
     arg_string = ""
     if hasattr(argument, 'md5hash'):
         return argument.md5hash
     if hasattr(argument, 'xxhash64'):
         return argument.xxhash64
     if type(argument) is numpy.ndarray:
         if argument.size > 181440000:
             return self.__hash_choice(argument.data)
         else:
             return str(xxhash.xxh64(argument.data))
     if type(argument) is pandas.core.frame.DataFrame:
         col_values_list = list(argument.columns.values)
         try:
             col_values_string = ''.join(col_values_list)
             arg_string = col_values_string
             if argument.values.size > 181440000:
                 return str(xxh.hash64(argument.data)) + "+" + str(xxhash.xxh64(arg_string))
             else:
                 return self.__hash_choice(argument.values.data) + "+" + str(xxhash.xxh64(arg_string))
         except:
             if argument.values.size > 181440000:
                 return str(xxh.hash64(argument.values.data))
             else:
                 return self.__hash_choice(argument.values.data)
     if type(argument) is list or type(argument) is tuple:
         arg_string = str(len(argument))
     arg_string += str(argument)
     return self.__hash_choice(arg_string)

def hashRequests(authTicket, payload):
    baseHash = xxhash.xxh64(
        authTicket.SerializeToString(),
        seed=0x1B845238
    ).intdigest()

    # Serialize and hash each request
    return [xxhash.xxh64(
        request.SerializeToString(),
        seed=baseHash
    ).intdigest() for request in payload]

def string_hash(value,length=11):
    s = ''
    for i in range(0,length,11):
        s = s + xxhash.xxh64(value+str(i)).hexdigest()
    s = encode_hash(int(s,16))[:length]
    if len(s) < length:
        s = s + "A" * (length - len(s))
    return s

def string_hash_bits(value,length_in_bits=128):
    ''' Length must be a multiple of 4'''
    hex_length = length_in_bits / 4
    s = ''
    for i in range(0,length_in_bits,64):
        s = s + xxhash.xxh64(value+str(i)).hexdigest()
    s = s[:hex_length]
    x = int(s,16)
    return x

def subscribe(request):
    """
    Subcribe the given email to the given URL.

    TODO BEN: Include Subscription title or description in POST variables
    """
    url = request.POST['subscription_url']
    email = request.POST['email']

    user, created_user = User.objects.get_or_create(email=email)
    if created_user:
        user_verification_hash = uuid.uuid4().hex
        user_verification = Verification.objects.create(
            verified=False,
            verification_hash=user_verification_hash)
        user.verification = user_verification
        user.save()

    content, created_content = SubscribedContent.objects.get_or_create(url=url)
    if created_content:
        content_response = requests.get(url)
        content_hash = xxhash.xxh64(content_response.text).hexdigest()
        content.latest_content_hash = content_hash

    if not Subscription.objects.filter(user=user, content=content).exists():
        verification_hash = uuid.uuid4().hex
        verification_url = request.build_absolute_uri(
            reverse(
                'verify',
                kwargs={
                    'email': email,
                    'key': verification_hash}
            )
        )
        verification_item = Verification.objects.create(
            verified=False,
            verification_hash=verification_hash
        )

        Subscription.objects.create(
            user=user,
            content=content,
            verification=verification_item
        )

        email_sent = EMAIL.VERIFY_SUBSCRIPTION.send(
            email, {'verification_url': verification_url})
        if email_sent:
            message = MESSAGES.EMAIL.VERIFICATION_SENT.format(email)
        else:
            message = MESSAGES.EMAIL.ERROR_SENDING_EMAIL.format(
                email)
    else:
        message = MESSAGES.EMAIL.ALREADY_SUBSCRIBED

    messages.add_message(request, messages.INFO, message)
    return redirect(request.META['HTTP_REFERER'], {'message': message})

def xxhash64(path, block_size=4096):
    try:
        with open(path, 'rb') as rf:
            h = xxhash.xxh64()
            for chunk in iter(lambda: rf.read(block_size), b''):
                h.update(chunk)
        return h.hexdigest(), path
    except IOError:
        return None, path

    def hash(self):
        """Return hash of motif.

        This is an unique identifier of a motif, regardless of the id.

        Returns:
        hash : str
        """
        return xxhash.xxh64(self._pwm_to_str(3)).hexdigest()

    def test_XXH64_reset(self):
        x = xxhash.xxh64()
        h = x.intdigest()

        for i in range(10, 50):
            x.update(os.urandom(i))

        x.reset()

        self.assertEqual(h, x.intdigest())

 def save(self, *args, **kwargs):
     new_hash = xxhash.xxh64(self.content_raw).hexdigest()
     mentioned_users = []
     if new_hash != self.raw_content_hash or (not self.pk):
         # To (re-)render the content if content changed or topic is newly created
         self.content_rendered, mentioned_users = render_content(self.content_raw, sender=self.user.username)
     super(Topic, self).save(*args, **kwargs)
     self.raw_content_hash = new_hash
     for to in mentioned_users:
             notify.delay(to=to.username, sender=self.user.username, topic=self.pk)

def xxhash_file(srcfile, logger, block_size=2**20):
    f_name = func_name()
    logger.info(f_name+"\t\tCalculating xx-hash on : "+srcfile)
    f = open(srcfile, 'r')  
    x = xxhash.xxh64()
    while True:
        data = f.read(block_size)
        if not data:
            break
        x.update(data)
    return  x.hexdigest()

    def cached_parse_dhcp(self, lines, cur_time=None):
        if cur_time is None:
            cur_time = dt.utcnow()
        m = xxhash.xxh64()
        m.update("".join(lines[:self.dhcp_cache_len]).encode("utf8"))
        new_hash = m.digest()
        # new_len = len(lines)

        if new_hash != self.dhcp_hash:
            self.dhcp_cache_len = 0
            self.dhcp_cache = []
            m = xxhash.xxh64()

        lines = lines[self.dhcp_cache_len:]
        self.dhcp_cache.extend(self.from_dhcp(lines, cur_time))
        m.update("".join(lines).encode("utf8"))
        self.dhcp_hash = m.digest()
        self.dhcp_cache_len += len(lines)

        return self.dhcp_cache

    def generate_content_hashes(self):
        """
        Generate a dictionary which maps parl_ids to their respective hashes

        Used for speedy comparison of changes
        """
        es_response = json.loads(self.get_content())
        content_hashes = {}
        for res in es_response['result']:
            content_hashes[res['parl_id']] = xxhash.xxh64(
                json.dumps(res)).hexdigest()
        return json.dumps(content_hashes)

 def memoize_wrapper(*args, **kwargs):
     hash = xxhash.xxh64(str(args) + str(kwargs)).intdigest()
     path = path_pattern.format(hash=hash)
     try:
         with open(path, 'rb') as file:
             logger.debug("Loading pickle %s", path)
             data = pickle.load(file)
     except (FileNotFoundError, EOFError):
         data = fn(*args, **kwargs)
         with open(path, 'wb') as file:
             pickle.dump(data, file)
     return data