Python mmh3.hash方法代码示例

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def groupby(from_dtm, to_dtm, tmp_dir, out_path, num_chunks=10):
    from_dtm, to_dtm = map(str, [from_dtm, to_dtm])
    fouts = {idx: open(os.path.join(tmp_dir, str(idx)), 'w')
             for idx in range(num_chunks)}
    files = sorted([path for path, _ in iterate_data_files(from_dtm, to_dtm)])
    for path in tqdm.tqdm(files, mininterval=1):
        for line in open(path):
            user = line.strip().split()[0]
            chunk_index = mmh3.hash(user, 17) % num_chunks
            fouts[chunk_index].write(line)

    map(lambda x: x.close(), fouts.values())
    with open(out_path, 'w') as fout:
        for chunk_idx in fouts.keys():
            _groupby = {}
            chunk_path = os.path.join(tmp_dir, str(chunk_idx))
            for line in open(chunk_path):
                tkns = line.strip().split()
                userid, seen = tkns[0], tkns[1:]
                _groupby.setdefault(userid, []).extend(seen)
            os.remove(chunk_path)
            for userid, seen in six.iteritems(_groupby):
                fout.write('%s %s\n' % (userid, ' '.join(seen))) 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def queryshodan(url):
    o = urlparse(url)
    if len(o.path)>=2:
        url = url
    else:
        url = url+"/favicon.ico"
    try:
        hash = getfaviconhash(url)
        if hash:
            query = "http.favicon.hash:{}".format(hash)
            count = api.count(query)['total']
            if count == 0:
                print("[-] No result")
            else:
                print("[+] Try to get {} ip.".format(count))
                for hosts in api.search_cursor(query):
                    print("[+] Get ip: "+hosts['ip_str'])
        else:
            print("[!] No icon find.")
    except Exception:
        print("[!] Invalid API key")
    except KeyboardInterrupt, e:
        print("[*] Shutting down...") 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def mm3hash_float(name):
    hash_32 = mmh3.hash(name)
    if hash_32 < 0:
        hash_32 = (-hash_32 - 1) ^ 0xFFFFFFFF

    mantissa = hash_32 & ((1 << 23) - 1)
    exp = (hash_32 >> 23) & ((1 << 8) - 1)
    exp = max(exp, 1)
    exp = min(exp, 254)
    exp =  exp << 23
    sign = (hash_32 >> 31)
    float_bits = exp | mantissa
    packed = struct.pack('=I', float_bits)
    packed = '\0' * (4 - len(packed)) + packed     # packed must be exactly 4 long
    if sign == 1:
        return -struct.unpack('=f', packed)[0]
    elif sign == 0:
        return struct.unpack('=f', packed)[0] 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def test_manifest(self):
        """Testing function to check for implementation errors and hash collisions.
        Checks all names and values in the manifest in the manifest by rehashing them,
        to ensure that the entire process is sound. Also finds collisions. Returns a tuple
        of errors and collisions.
        """
        self.parse_manifest()

        ids = {}
        errors = []
        collisions = []
        manifest = self.cryptomattes[self.selection]["names_to_IDs"]
        for name, idvalue in manifest.iteritems():
            if mm3hash_float(name) != idvalue:
                errors.append("computed ID doesn't match manifest ID: (%s, %s)" % (idvalue, mm3hash_float(name)))
            else:
                if idvalue in ids:
                    collisions.append("colliding: %s %s" % (ids[idvalue], name))
                ids[idvalue] = name

        print "Tested %s, %s names" % (self.nuke_node.name(), len(manifest))
        print "    ", len(errors), "non-matching IDs between python and c++."
        print "    ", len(collisions), "hash collisions in manifest."

        return errors, collisions 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def Calculate(self, string):
        if self.name == "md5":
            hash = hashlib.md5(string).hexdigest()

        elif self.name == "sha1":
            hash = hashlib.sha1(string).hexdigest()

        elif self.name == "crc":
            crc32 = crcmod.Crc(0x104c11db7, initCrc=0, xorOut=0xFFFFFFFF)
            crc32.update(string)
            hash = crc32.hexdigest()

        elif self.name == "murmur":
            hash = mmh3.hash(string)

        elif self.name == "ssdeep":
            hash = ssdeep.hash(string)

        elif self.name == "tlsh":
            hash = tlsh.hash(string)

        return hash 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def n_feature_hash(feature, dims, seeds):
    """N-hot-encoded feature hashing.

    Args:
        feature (str): Target feature represented as string.
        dims (list of int): Number of dimensions for each hash value.
        seeds (list of float): Seed of each hash function (mmh3).

    Returns:
        numpy 1d array: n-hot-encoded feature vector for `s`.

    """
    vec = np.zeros(sum(dims))
    offset = 0

    for seed, dim in zip(seeds, dims):
        vec[offset:(offset + dim)] = feature_hash(feature, dim, seed)
        offset += dim

    return vec 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def feature_hash(feature, dim, seed=123):
    """Feature hashing.

    Args:
        feature (str): Target feature represented as string.
        dim (int): Number of dimensions for a hash value.
        seed (float): Seed of a MurmurHash3 hash function.

    Returns:
        numpy 1d array: one-hot-encoded feature vector for `s`.

    """
    vec = np.zeros(dim)
    i = mmh3.hash(feature, seed) % dim
    vec[i] = 1
    return vec 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def multiple_feature_hash(feature, dim, seed=123):
    """Feature hashing using multiple hash function.
    This technique is effective to prevent collisions.

    Args:
        feature (str): Target feature represented as string.
        dim (int): Number of dimensions for a hash value.
        seed (float): Seed of a MurmurHash3 hash function.

    Returns:
        numpy 1d array: one-hot-encoded feature vector for `s`.

    """
    vec = np.zeros(dim)
    i = mmh3.hash(feature, seed) % dim
    vec[i] = 1 if mmh3.hash(feature) % 2 else -1
    return vec 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def hash_int64_array(array, seed=MMH3_SEED):
    """Hash an int64 array into a 32-bit integer.

    Parameters
    ----------
    array : ndarray of int64
        Numpy array containing integers
    seed : any, optional
        Seed for MurmurHash3.

    Returns
    -------
    int : 32-bit integer
    """
    if not np.issubdtype(array.dtype, IDENT_DTYPE):
        raise TypeError(
            "Provided array has dtype {} not {}".format(
                array.dtype, IDENT_DTYPE.__name__
            )
        )
    # ensure all hashed integers are positive
    hashed_int = mmh3.hash(array, seed)
    return hashed_int 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def num_hashes(self):
        '''The number of hash functions (k) given m and n, minimizing p.

        This method returns the number of times (k) to run our hash functions
        on a given input string to compute bit offsets into the underlying
        string representing this Bloom filter.  m is the size in bits of the
        underlying string, n is the number of elements that you expect to
        insert, and p is your acceptable false positive probability.

        More about the formula that this method implements:
            https://en.wikipedia.org/wiki/Bloom_filter#Optimal_number_of_hash_functions
        '''
        try:
            return self._num_hashes
        except AttributeError:
            self._num_hashes = self.size() / self.num_values * math.log(2)
            self._num_hashes = math.ceil(self._num_hashes)
            return self.num_hashes() 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def mmh3_hash_function(x):
    return mmh3.hash(x, 42, signed=True) 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def _get_redis_shard(self, key):
        """

        Args:
            key (str): The key to hash on to calculate the shard to get data from/to

        Returns:
            redis.StrictRedis: The Redis Connection
        """
        shard_no = mmh3.hash(key, signed=False) % self._no_of_shards
        return self._panoptes_context.get_redis_connection(group=self.redis_group, shard=shard_no) 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def getfaviconhash(url):
    try:
        response = requests.get(url)
        if response.headers['Content-Type'] == "image/x-icon":
            favicon = response.content.encode('base64')
            hash = mmh3.hash(favicon)
        else:
            hash = None
    except:
        print("[!] Request Error")
        hash = None
    return hash 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def parse_data(self, label, h, i):
        Y = self.y_vocab.get(label)
        if Y is None and self.div in ['dev', 'test']:
            Y = 0
        if Y is None and self.div != 'test':
            return [None] * 2
        Y = to_categorical(Y, len(self.y_vocab))

        product = h['product'][i]
        if six.PY3:
            product = product.decode('utf-8')
        product = re_sc.sub(' ', product).strip().split()
        words = [w.strip() for w in product]
        words = [w for w in words
                 if len(w) >= opt.min_word_length and len(w) < opt.max_word_length]
        if not words:
            return [None] * 2

        hash_func = hash if six.PY2 else lambda x: mmh3.hash(x, seed=17)
        x = [hash_func(w) % opt.unigram_hash_size + 1 for w in words]
        xv = Counter(x).most_common(opt.max_len)

        x = np.zeros(opt.max_len, dtype=np.float32)
        v = np.zeros(opt.max_len, dtype=np.int32)
        for i in range(len(xv)):
            x[i] = xv[i][0]
            v[i] = xv[i][1]
        return Y, (x, v) 

# 需要导入模块: import mmh3 [as 别名]
# 或者: from mmh3 import hash [as 别名]
def set_payload(self, payload, write=True):
        self.set_payload_len(len(payload), write=False)
        self.set_payload_hash(mmh3.hash(payload), write)
        atomic_write(QueueNode.__get_payload_filename(self.get_exit_reason(), self.get_id()), payload)