Java Snappy类代码示例

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public static byte[] compressSnappy(final BlockValue bv, final FTableDescriptor td) {
  byte[] bytes = null;
  final int totalSize = bv.getValueSize() + (bv.size() * 4);
  try (HeapDataOutputStream hdos = new HeapDataOutputStream(totalSize, null, true)) {
    for (final byte[] rec : bv.getRecords()) {
      hdos.writeInt(rec.length);
      hdos.write(rec);
    }
    // bytes = Snappy.compress(hdos.toByteArray());
    bytes = hdos.toByteArray();
    byte[] compressedOut = new byte[Snappy.maxCompressedLength(bytes.length)];
    int compressedSize = Snappy.compress(bytes, 0, bytes.length, compressedOut, 0);
    bytes = Arrays.copyOf(compressedOut, compressedSize);
  } catch (IOException e) {
    logger.error("Error converting to Snappy format.", e);
  }
  return bytes;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public static int writeCompressedWithoutOriginalBufferLength(@NotNull DataOutput out, @NotNull byte[] bytes, int length) throws IOException {
  long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;

  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
  int compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && requests % 1000  == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l  / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@NotNull
public static byte[] readCompressedWithoutOriginalBufferLength(@NotNull DataInput in) throws IOException {
  int size = DataInputOutputUtil.readINT(in);

  byte[] bytes = spareBufferLocal.getBuffer(size);
  in.readFully(bytes, 0, size);

  int decompressedRequests = myDecompressionRequests.incrementAndGet();
  long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;

  byte[] decompressedResult = Snappy.uncompress(bytes, 0, size);

  long doneTime = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
  long decompressedSize = myDecompressedSize.addAndGet(size);
  long decompressedTime = myDecompressionTime.addAndGet(doneTime);
  if (DUMP_COMPRESSION_STATS && decompressedRequests % 1000 == 0) {
    System.out.println("Decompressed " + decompressedRequests + " times, size: " + decompressedSize  + " for " + (decompressedTime / 1000000) + "ms");
  }

  return decompressedResult;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@NotNull
public static Object compressCharSequence(@NotNull CharSequence string, @NotNull Charset charset) {
  if (string.length() < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    return Snappy.compress(string.toString().getBytes(charset));
  }
  catch (CorruptionException ex) {
    ex.printStackTrace();
    return string;
  }
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public static byte[] decompressBytes(byte[] bytesArray) throws ClientException  { 
	
	byte[] checkSumBuf = new byte[8];
	checkSumBuf[0] = bytesArray[bytesArray.length-8];
	checkSumBuf[1] = bytesArray[bytesArray.length-7];
	checkSumBuf[2] = bytesArray[bytesArray.length-6];
	checkSumBuf[3] = bytesArray[bytesArray.length-5];
	checkSumBuf[4] = bytesArray[bytesArray.length-4];
	checkSumBuf[5] = bytesArray[bytesArray.length-3];
	checkSumBuf[6] = bytesArray[bytesArray.length-2];
	checkSumBuf[7] = bytesArray[bytesArray.length-1];
	
	
	ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
    buffer.put(checkSumBuf);
    buffer.flip();//need flip 
    long checkSum = buffer.getLong();
    
    Adler32 adler32 = new Adler32();
	adler32.update(bytesArray, 0, bytesArray.length-8);
	if(checkSum !=adler32.getValue())
		throw new ClientException("Data corruption detected - checksum failure. Please, try again.");
    
	return Snappy.uncompress(bytesArray, 0, bytesArray.length -8 );
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@Test
public void shouldCompressSmallContent() {
    String text = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo " +
        "ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient " +
        "montes, nascetur ridiculus mus.";

    channel.pipeline().addFirst(new SnappyFeatureHandler());

    ByteBuf content = Unpooled.copiedBuffer(text, CharsetUtil.UTF_8);

    UpsertRequest request = new UpsertRequest("key", content.copy(), "bucket");
    request.partition((short) 512);
    channel.writeOutbound(request);
    FullBinaryMemcacheRequest outbound = (FullBinaryMemcacheRequest) channel.readOutbound();
    assertNotNull(outbound);

    assertEquals(KeyValueHandler.DATATYPE_SNAPPY, outbound.getDataType());

    byte[] compressed = new byte[outbound.content().readableBytes()];
    outbound.content().getBytes(0, compressed);
    byte[] uncompressed = Snappy.uncompress(compressed, 0, compressed.length);
    assertArrayEquals(text.getBytes(CharsetUtil.UTF_8), uncompressed);
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@Test
public void shouldDecompressSmallContent() {
    String text = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo " +
        "ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient " +
        "montes, nascetur ridiculus mus.";

    channel.pipeline().addFirst(new SnappyFeatureHandler());

    ByteBuf content = Unpooled.wrappedBuffer(Snappy.compress(text.getBytes(CharsetUtil.UTF_8)));
    FullBinaryMemcacheResponse response = new DefaultFullBinaryMemcacheResponse(KEY, Unpooled.EMPTY_BUFFER,
        content.copy());
    response.setDataType(KeyValueHandler.DATATYPE_SNAPPY);

    GetRequest requestMock = mock(GetRequest.class);
    requestQueue.add(requestMock);
    channel.writeInbound(response);

    GetResponse event = (GetResponse) eventSink.responseEvents().get(0).getMessage();
    assertEquals(text, event.content().toString(CHARSET));
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public boolean set(final String key, final String value) throws InterruptedException, ExecutionException, UnsupportedEncodingException,
		MasterMayBeLostException {
	byte[] b = value.getBytes("utf-8");
	b = Snappy.compress(b);// ѹ��
	Add add = Add.newBuilder().setKey(key).setValue(ByteString.copyFrom(b)).build();
	Future<byte[]> future = operator.add(add.toByteArray(), CommandType.USER_WRITE);
	try {
		byte[] response = future.get(5, TimeUnit.SECONDS);
		if (response[0] == 1) {
			return true;
		} else {
			return false;
		}
	} catch (TimeoutException e) {
		throw new MasterMayBeLostException();
	}
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public int compress(byte[] value, int offset) {
	int size;
	if (end > start) {
		value[offset] = 0;
		size = Snappy.compress(buffer, start, end - start, value,
				offset + 1);
	} else {
		value[offset] = 1;
		size = Snappy.compress(buffer, start, buffer.length - start, value,
				offset + 5);
		Utils.encodeInt(value, offset + 1, size);
		size += 4 + Snappy.compress(buffer, 0, end, value, offset + 5
				+ size);
	}
	return size + 1;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public void decompress(byte[] value, int offset, int len) {
	if (value[offset] == 0) {
		grow(Snappy.getUncompressedLength(value, offset + 1));
		end = Snappy.uncompress(value, offset + 1, len - 1, buffer, 0);
	} else {
		int s = Utils.decodeInt(value, offset + 1);
		// Calculate uncompress size
		offset += 5;
		len -= 5;
		int un_s = Snappy.getUncompressedLength(value, offset);
		un_s += Snappy.getUncompressedLength(value, offset + s);
		grow(un_s);

		end = Snappy.uncompress(value, offset, s, buffer, 0);
		end += Snappy.uncompress(value, offset + s, len - s, buffer, end);
	}
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
public static int writeCompressedWithoutOriginalBufferLength(@Nonnull DataOutput out, @Nonnull byte[] bytes, int length) throws IOException {
  long started = System.nanoTime();

  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
  int compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = System.nanoTime() - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && requests % 1000  == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l  / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@Nonnull
public static byte[] readCompressedWithoutOriginalBufferLength(@Nonnull DataInput in) throws IOException {
  int size = DataInputOutputUtil.readINT(in);

  byte[] bytes = spareBufferLocal.getBuffer(size);
  in.readFully(bytes, 0, size);

  int decompressedRequests = myDecompressionRequests.incrementAndGet();
  long started = System.nanoTime();

  byte[] decompressedResult = Snappy.uncompress(bytes, 0, size);

  long doneTime = System.nanoTime() - started;
  long decompressedSize = myDecompressedSize.addAndGet(size);
  long decompressedTime = myDecompressionTime.addAndGet(doneTime);
  if (DUMP_COMPRESSION_STATS && decompressedRequests % 1000 == 0) {
    System.out.println("Decompressed " + decompressedRequests + " times, size: " + decompressedSize  + " for " + (decompressedTime / 1000000) + "ms");
  }

  return decompressedResult;
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@Nonnull
public static Object compressCharSequence(@Nonnull CharSequence string, @Nonnull Charset charset) {
  if (string.length() < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    return Snappy.compress(string.toString().getBytes(charset));
  }
  catch (CorruptionException ex) {
    ex.printStackTrace();
    return string;
  }
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
@Override
public byte[] decompress(byte[] input) {
  try {
    return Snappy.uncompress(input, 0, input.length);
  } catch (CorruptionException e) {
    throw new CompressionException(e);
  }
}
 

import org.iq80.snappy.Snappy; //导入依赖的package包/类
private byte[] getMaybeCompressedBytes(@Nullable byte[] bytes) throws SingularityTranscoderException {

    if (bytes == null || bytes.length == 0) {
      return bytes;
    }

    return compressLargeDataObjects ? Snappy.compress(bytes) : bytes;
  }