Java Snappy.compress方法代码示例

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compressSnappy(final BlockValue bv, final FTableDescriptor td) {
  byte[] bytes = null;
  final int totalSize = bv.getValueSize() + (bv.size() * 4);
  try (HeapDataOutputStream hdos = new HeapDataOutputStream(totalSize, null, true)) {
    for (final byte[] rec : bv.getRecords()) {
      hdos.writeInt(rec.length);
      hdos.write(rec);
    }
    // bytes = Snappy.compress(hdos.toByteArray());
    bytes = hdos.toByteArray();
    byte[] compressedOut = new byte[Snappy.maxCompressedLength(bytes.length)];
    int compressedSize = Snappy.compress(bytes, 0, bytes.length, compressedOut, 0);
    bytes = Arrays.copyOf(compressedOut, compressedSize);
  } catch (IOException e) {
    logger.error("Error converting to Snappy format.", e);
  }
  return bytes;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public static int writeCompressedWithoutOriginalBufferLength(@NotNull DataOutput out, @NotNull byte[] bytes, int length) throws IOException {
  long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;

  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
  int compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && requests % 1000  == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l  / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
@NotNull
public static Object compressCharSequence(@NotNull CharSequence string, @NotNull Charset charset) {
  if (string.length() < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    return Snappy.compress(string.toString().getBytes(charset));
  }
  catch (CorruptionException ex) {
    ex.printStackTrace();
    return string;
  }
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public boolean set(final String key, final String value) throws InterruptedException, ExecutionException, UnsupportedEncodingException,
		MasterMayBeLostException {
	byte[] b = value.getBytes("utf-8");
	b = Snappy.compress(b);// ѹ��
	Add add = Add.newBuilder().setKey(key).setValue(ByteString.copyFrom(b)).build();
	Future<byte[]> future = operator.add(add.toByteArray(), CommandType.USER_WRITE);
	try {
		byte[] response = future.get(5, TimeUnit.SECONDS);
		if (response[0] == 1) {
			return true;
		} else {
			return false;
		}
	} catch (TimeoutException e) {
		throw new MasterMayBeLostException();
	}
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public int compress(byte[] value, int offset) {
	int size;
	if (end > start) {
		value[offset] = 0;
		size = Snappy.compress(buffer, start, end - start, value,
				offset + 1);
	} else {
		value[offset] = 1;
		size = Snappy.compress(buffer, start, buffer.length - start, value,
				offset + 5);
		Utils.encodeInt(value, offset + 1, size);
		size += 4 + Snappy.compress(buffer, 0, end, value, offset + 5
				+ size);
	}
	return size + 1;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public static int writeCompressedWithoutOriginalBufferLength(@Nonnull DataOutput out, @Nonnull byte[] bytes, int length) throws IOException {
  long started = System.nanoTime();

  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));
  int compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = System.nanoTime() - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && requests % 1000  == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l  / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
@Nonnull
public static Object compressCharSequence(@Nonnull CharSequence string, @Nonnull Charset charset) {
  if (string.length() < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    return Snappy.compress(string.toString().getBytes(charset));
  }
  catch (CorruptionException ex) {
    ex.printStackTrace();
    return string;
  }
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
private byte[] getMaybeCompressedBytes(@Nullable byte[] bytes) throws SingularityTranscoderException {

    if (bytes == null || bytes.length == 0) {
      return bytes;
    }

    return compressLargeDataObjects ? Snappy.compress(bytes) : bytes;
  }
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
private void writeBlock(BlockBuilder block, TableFormat.BlockHandle handle) throws IOException {
    Slice raw = block.finish();
    Slice blockContents;
    int type = options.compression;
    switch (type) {
        case Options.CompressionType.NO_COMPRESSION:
            blockContents = raw;
            break;

        case Options.CompressionType.SNAPPY_COMPRESSION:
            int maxLength = Snappy.maxCompressedLength(raw.data().length);
            if (maxLength > compressBuffer.length) {
                compressBuffer = new byte[maxLength * 2];
            }
            int compressLength = Snappy.compress(raw.data(), 0, raw.size(), compressBuffer, 0);
            if (compressLength < raw.size() - raw.size() / 8) {
                blockContents = new Slice(Arrays.copyOf(compressBuffer, compressLength));
            } else {
                blockContents = raw;
                type = Options.CompressionType.NO_COMPRESSION;
            }
            break;
        default:
            throw new IOException("Compression type not supported");
    }
    writeRawBlock(blockContents, type, handle);
    block.reset();
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public static int writeCompressed(@NotNull DataOutput out, @NotNull byte[] bytes, int length) throws IOException {
  if (length > COMPRESSION_THRESHOLD) {
    final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(Snappy.maxCompressedLength(length));

    int compressedSize = Snappy.compress(bytes, 0, length, compressedOutputBuffer, 0);
    DataInputOutputUtil.writeINT(out, -compressedSize);
    out.write(compressedOutputBuffer, 0, compressedSize);
    return compressedSize;
  }
  else {
    DataInputOutputUtil.writeINT(out, length);
    out.write(bytes, 0, length);
    return length;
  }
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
@NotNull
public static Object compressStringRawBytes(@NotNull CharSequence string) {
  int length = string.length();
  if (length < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    ByteArrayOutputStream bytes = new ByteArrayOutputStream(length);
    @NotNull DataOutput out = new DataOutputStream(bytes);

    DataInputOutputUtil.writeINT(out, length);
    for (int i=0; i< length;i++) {
      char c = string.charAt(i);
      DataInputOutputUtil.writeINT(out, c);
    }
    byte[] compressedBytes = Snappy.compress(bytes.toByteArray());
    return compressedBytes.length < length * 2 ? compressedBytes : string;
  }
  catch (CorruptionException ex) {
    ex.printStackTrace();
    return string;
  }
  catch (IOException e) {
    e.printStackTrace();
    return string;
  }
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compressBytes(byte[] bytesArray) { 
	
	
	int decompressedLength = bytesArray.length;	
	int maxCompressedLength = Snappy.maxCompressedLength(decompressedLength);
	byte[] compressed = new byte[maxCompressedLength];
	int compressedLength = Snappy.compress(bytesArray, 0, decompressedLength, compressed, 0);
	byte[] truncated = Arrays.copyOfRange(compressed, 0, compressedLength );
	byte[] checkSumExt = Arrays.copyOf(truncated, truncated.length+8);
	
	Adler32 adler32 = new Adler32();
	adler32.update(truncated);
	long checkSum =adler32.getValue();
	ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
	byte[] checkSumBuf  =buffer.putLong(checkSum).array();
	
	checkSumExt[truncated.length] = checkSumBuf[0];
	checkSumExt[truncated.length+1] = checkSumBuf[1];
	checkSumExt[truncated.length+2] = checkSumBuf[2];
	checkSumExt[truncated.length+3] = checkSumBuf[3];
	checkSumExt[truncated.length+4] = checkSumBuf[4];
	checkSumExt[truncated.length+5] = checkSumBuf[5];
	checkSumExt[truncated.length+6] = checkSumBuf[6];
	checkSumExt[truncated.length+7] = checkSumBuf[7];
	
	
	return checkSumExt;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
@Override
public ByteBuffer compress(ByteBuffer in) throws IOException {
  ByteBuffer out =
      ByteBuffer.allocate(Snappy.maxCompressedLength(in.remaining())+4);
  int size = Snappy.compress(in.array(), in.position(), in.remaining(),
      out.array(), 0);
  crc32.reset();
  crc32.update(in.array(), in.position(), in.remaining());
  out.putInt(size, (int)crc32.getValue());

  out.limit(size+4);

  return out;
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
@Override
public byte[] deflate(InputStream data) throws CompressionException
{
  try
  {
    byte[] temp = IOUtils.toByteArray(data);
    return Snappy.compress(temp);
  }
  catch (IOException e)
  {
    throw new CompressionException(CompressionConstants.DECODING_ERROR + getContentEncodingName(), e);
  }
}
 

import org.iq80.snappy.Snappy; //导入方法依赖的package包/类
/**
 * Compress the given array of bytes.
 */
protected byte[] compress(byte[] in) {
    if (in == null) {
        throw new NullPointerException("Can't compress null");
    }

    byte[] compressed;
    try {
        compressed = Snappy.compress(in);
    } catch (Exception e) {
        throw new RuntimeException("IO exception compressing data", e);
    }
    getLogger().debug("Compressed %d bytes to %d", in.length, compressed.length);
    return compressed;
}