Java Hashing类代码示例

import com.google.common.hash.Hashing; //导入依赖的package包/类
public static String genSignature(HttpServletRequestEx requestEx) {
  Hasher hasher = Hashing.sha256().newHasher();
  hasher.putString(requestEx.getRequestURI(), StandardCharsets.UTF_8);
  for (String paramName : paramNames) {
    String paramValue = requestEx.getHeader(paramName);
    if (paramValue != null) {
      hasher.putString(paramName, StandardCharsets.UTF_8);
      hasher.putString(paramValue, StandardCharsets.UTF_8);
      System.out.printf("%s %s\n", paramName, paramValue);
    }
  }

  byte[] bytes = requestEx.getBodyBytes();
  if (bytes != null) {
    hasher.putBytes(bytes, 0, requestEx.getBodyBytesLength());
  }

  return hasher.hash().toString();
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
public boolean shouldReload() {
    if(context == null) return true;
    if(!configuration.autoReload()) return false;
    if(context.loadedFiles().isEmpty()) return configuration.reloadWhenError();

    try {
        for(Map.Entry<Path, HashCode> loaded : context.loadedFiles().entrySet()) {
            HashCode latest = Files.hash(loaded.getKey().toFile(), Hashing.sha256());
            if(!latest.equals(loaded.getValue())) return true;
        }

        return false;
    } catch (IOException e) {
        return true;
    }
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
@Override
public Server choose(Object key) {
	
	//使用一致性哈希进行分发
	//TODO 此处使用了GUAVA中简单的一致性哈希算法选择服务，但这里存在性能缺陷：当reachableServers中间某一个服务节点失效了
	//那么后续节点的一致性哈希结果将会不匹配，后续需要使用更完善的哈希环 加上 虚拟节点 的形式解决本问题
	List<Server> reachableServers = getLoadBalancer().getReachableServers();
	if(reachableServers != null && reachableServers.size() != 0){
		int serverSeq = Hashing.consistentHash(Thread.currentThread().getId(), reachableServers.size());
		return reachableServers.get(serverSeq);
	} else {
		return super.choose(key);
	}
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
public boolean isFullDuplicate(IHttpRequestResponse messageInfo) {
    PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true);
    IResponseInfo respInfo = helpers.analyzeResponse(messageInfo.getResponse());

    if (dubBloomFilter == null) return false;

    HashFunction m_hash = Hashing.murmur3_32();
    if (helpers.bytesToString(messageInfo.getResponse()).length() > respInfo.getBodyOffset()) {
        String body = helpers.bytesToString(messageInfo.getResponse()).substring(respInfo.getBodyOffset());

        /* full-dub detection */
        String dedupHashValue = m_hash.hashBytes(helpers.stringToBytes(body)).toString();
        if (dubBloomFilter.mightContain(dedupHashValue)) {
            return true;
        }
        dubBloomFilter.put(dedupHashValue);
    }

    return false;
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
private static String makeSign(long userId, long sid, long expireTime) {
    StringBuilder stringBuilder = new StringBuilder();
    long type = (userId + sid + expireTime) % 3;
    if (type == 0) {
        stringBuilder.append(expireTime);
        stringBuilder.append(userId);
        stringBuilder.append(sid);
        stringBuilder.append(type);
    } else if (type == 1) {
        stringBuilder.append(userId);
        stringBuilder.append(expireTime);
        stringBuilder.append(sid);
        stringBuilder.append(type);
    } else {
        stringBuilder.append(sid);
        stringBuilder.append(expireTime);
        stringBuilder.append(userId);
        stringBuilder.append(type);
    }

    return Hashing.sha256().hashBytes(stringBuilder.toString().getBytes()).toString();
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
/**
 * Calculate bit positions of {@code str}.
 * <p>
 * See "Less Hashing, Same Performance: Building a Better Bloom Filter" by Adam Kirsch and Michael
 * Mitzenmacher.
 * </p>
 *
 * @param str
 * @return
 */
public int[] calcBitPositions(String str) {
    int[] bitPositions = new int[this.k];

    long hash64 = Hashing.murmur3_128().hashString(str, UTF_8).asLong();

    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);

    for (int i = 1; i <= this.k; i++) {
        int combinedHash = hash1 + (i * hash2);
        // Flip all the bits if it's negative (guaranteed positive number)
        if (combinedHash < 0) {
            combinedHash = ~combinedHash;
        }
        bitPositions[i - 1] = combinedHash % this.m;
    }

    return bitPositions;
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
@Test
public void testImageTransform() throws Exception {
  ClassLoader loader = Thread.currentThread().getContextClassLoader();
  String origPath = loader.getResource("com/filestack/sample_image.jpg").getPath();
  File origFile = new File(origPath);

  FileLink fileLink = client.upload(origPath, false);
  HANDLES.add(fileLink.getHandle());

  ImageTransform transform = fileLink.imageTransform();
  transform.addTask(new CropTask(0, 0, 500, 500));

  String cropPath = loader.getResource("com/filestack/sample_image_cropped.jpg").getPath();
  File cropFile = new File(cropPath);

  String correct = Files.asByteSource(cropFile).hash(Hashing.sha256()).toString();
  byte[] bytes = transform.getContent().bytes();
  String output = Hashing.sha256().hashBytes(bytes).toString();

  Assert.assertEquals(correct, output);
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
private void processRegions(File file, RegionProcessor regionProcessor, String randomAccessFileMode, FileChannel.MapMode mapMode) throws IOException {
    try (
            RandomAccessFile randomAccessFile = new RandomAccessFile(file.getAbsolutePath().toFile(), randomAccessFileMode);
            FileChannel channel = randomAccessFile.getChannel()

    ) {
        RegionCalculator.calculateForSize(file, file.getSize());


        for (Region region : file.getRegions().values()) {
            Hasher hasher = Hashing.sha256().newHasher();
            MappedByteBuffer mappedByteBuffer = channel.map(mapMode, region.getOffset(), region.getSize());

            int sum = regionProcessor.processRegion(region, hasher, mappedByteBuffer);

            region.setQuickDigest(sum);

            byte[] slowDigest = hasher.hash().asBytes();
            region.setSlowDigest(slowDigest);

            clientMessageHandler.submitClientRegionMessage(clientId, file, region.getOffset(), region.getSize(), sum, slowDigest);
        }
    }
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
/**
 * Files exported from jars are exported into a certain folder so that we can rebuild them
 * when the related jar file changes.
 */
@NonNull
private static String getJarFilePrefix(@NonNull File inputFile) {
    // get the filename
    String name = inputFile.getName();
    // remove the extension
    int pos = name.lastIndexOf('.');
    if (pos != -1) {
        name = name.substring(0, pos);
    }

    // add a hash of the original file path.
    String input = inputFile.getAbsolutePath();
    HashFunction hashFunction = Hashing.sha1();
    HashCode hashCode = hashFunction.hashString(input, Charsets.UTF_16LE);

    return name + "-" + hashCode.toString();
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
public boolean put(String url) {
    byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes();
    long hash1 = this.lowerEight(bytes);
    long hash2 = this.upperEight(bytes);
    boolean bitsChanged = false;
    long combinedHash = hash1;
    try {
        for (int i = 0; i < numHashFunctions; ++i) {
        /* mask combinedHash with 0x7FFFFFFFFFFFFFFF */
            bitsChanged |= bits.set((combinedHash & 9223372036854775807L) % bitsSize);
            combinedHash += hash2;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return bitsChanged;
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
public boolean mightContain(String url) throws IOException {
    byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes();
    long hash1 = this.lowerEight(bytes);
    long hash2 = this.upperEight(bytes);
    long combinedHash = hash1;

    for (int i = 0; i < numHashFunctions; ++i) {
        if (!bits.get((combinedHash & 9223372036854775807L) % bitsSize)) {
            return false;
        }

        combinedHash += hash2;
    }

    return true;
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
@Override
public void process(ResultItems resultItems, Task task) {
    SpiderInfo spiderInfo = resultItems.get("spiderInfo");
    Webpage webpage = convertResultItems2Webpage(resultItems);
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.matchQuery("url", webpage.getUrl()));
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    if (response.getHits().totalHits() == 0) {
        try {
            client.prepareIndex(INDEX_NAME, TYPE_NAME)
                    .setId(Hashing.md5().hashString(webpage.getUrl(), Charset.forName("utf-8")).toString())
                    .setSource(gson.toJson(webpage))
                    .get();
        } catch (Exception e) {
            LOG.error("索引 Webpage 出错," + e.getLocalizedMessage());
        }
    }
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
/**
 * add
 *
 * @param element
 * @return
 */
public boolean add(CharSequence element) {
    if (element == null) {
        return false;
    }
    initCharset();
    BloomFilter<CharSequence> bf = this.bloomfilters.get(0);
    if (getBloomfilterBucketLength() > 1) {
        byte[] datas = element.toString().getBytes(this.charset);
        int bfIndex = Math.abs(Hashing.murmur3_128().hashBytes(datas).asInt()) % getBloomfilterBucketLength();
        bf = this.bloomfilters.get(bfIndex);
    }
    synchronized (bf) {
        bf.put(element);
        numberOfAddedElements++;
        return true;
    }
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
/**
 * 将Webmagic中的ResultItems转换为Webpage
 * @param items
 * @return
 */
public static Webpage convertToWebpage(ResultItems items) {
	if(items.toString().indexOf("ResultItems{fields={}") > 0) {
		return null;
	}
	Webpage page = new Webpage().setContent(items.get("content")).setTitle(items.get("title"))
			.setUrl(items.get("url")).setDomain(items.get("domain")).setSpiderInfoId(items.get("spiderInfoId"))
			.setGathertime(items.get("gathertime")).setSpiderUUID(items.get("spiderUUID"))
			.setKeywords(items.get("keywords")).setSummary(items.get("summary"))
			.setNamedEntity(items.get("namedEntity")).setPublishTime(items.get("publishTime"))
			.setCategory(items.get("category")).setRawHTML(items.get("rawHTML"))
			.setDynamicFields(items.get(DYNAMIC_FIELD)).setStaticFields(items.get("staticField"))
			.setAttachmentList(items.get("attachmentList")).setImageList(items.get("imageList"))
			.setProcessTime(items.get("processTime"));
	return 	page.setId(Hashing.sha256().hashString(page.getUrl(), Charset.forName("utf-8")).toString());
}
 

import com.google.common.hash.Hashing; //导入依赖的package包/类
@Override
public void process(ResultItems resultItems, Task task) {
	SpiderInfo info = resultItems.get("spiderInfo");
	Webpage page = convertToWebpage(resultItems);
	/*
	 * guava22.0不再对MD5()提供支持，
	 * 如果想更安全，使用sha256(), 
	 * 如果想更快，使用goodFastHash()
	 * */
	try {
		client.prepareIndex(INDEX_NAME, TYPE_NAME)
		   .setId(Hashing.sha256().hashString(page.getUrl(), Charset.forName("utf-8")).toString())
		   .setSource(GSON.toJson(page), XContentType.JSON)
		   .get();
	} catch(Exception e) {
		LOG.error("索引Webpage出错， 由于 " + e.getLocalizedMessage());
	}
}