Java Snappy.compress方法代码示例

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
/**
 * Test compression snappy raw.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
public void testCompressionSnappyRaw() throws IOException
{
	ByteBuffer src = ByteBuffer.allocateDirect(1024);
	ByteBuffer dst = ByteBuffer.allocateDirect(1024);
	String value = "0000000000000000000000000000000000000000000000000000000000000000000000000000000"+
       "0000000000000000000000000000000000000000000000000000000000000000000000000000000" +
       "0000000000000000000000000000000000000000000000000000000000000000000000000000000";
       src.put(value.getBytes());
       src.flip();
       int len = Snappy.compress(src, dst);
       LOG.info("Compressed size ="+len+" dst pos ="+dst.position()+" dst limit="+dst.limit());
       len = Snappy.uncompress(dst, src);
       LOG.info("Uncompressed size="+len);
       LOG.info("Original size="+value.length());
       
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public static ByteBuffer compressBufferForMessaging(ByteBuffer buffer) throws IOException {
    assert(buffer.isDirect());
    int lim = buffer.limit();
    int pos = buffer.position();
    int rem = buffer.remaining();
    IOBuffers buffers = getBuffersForCompression(buffer.remaining(), true);
    ByteBuffer output = buffers.output;

    final int compressedSize = Snappy.compress(buffer, output);
    ByteBuffer result = ByteBuffer.wrap(new byte[compressedSize + 4]);
    result.putInt(compressedSize);
    result.put(output);
    result.flip();
    
    return result;
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compress(byte[] uncompressData) {
    try {
        switch (CMP) {
            case SNAPPY:
                return Snappy.compress(uncompressData);
            case GZIP:
                ByteArrayOutputStream out = new ByteArrayOutputStream(1024);
                GZIPOutputStream gzout = new GZIPOutputStream(out);
                gzout.write(uncompressData, 0, uncompressData.length);
                gzout.close();
                return out.toByteArray();
            default:
                throw new RuntimeException("Should not happen!");
        }
    } catch (Exception e) {
        throw new IllegalStateException(e);
    }
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public Frame compress(Frame frame) throws IOException
{
    byte[] input = CBUtil.readRawBytes(frame.body);
    ByteBuf output = CBUtil.allocator.heapBuffer(Snappy.maxCompressedLength(input.length));

    try
    {
        int written = Snappy.compress(input, 0, input.length, output.array(), output.arrayOffset());
        output.writerIndex(written);
    }
    catch (final Throwable e)
    {
        output.release();
        throw e;
    }
    finally
    {
        //release the old frame
        frame.release();
    }

    return frame.with(output);
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
/**
 * Insert a tuple into the page, appending its serialized length to the file beforehand
 */
public void insert(MapTuple tuple){

  try {

    // Generate serial data and length
    byte[] serialData = Snappy.compress(Utils.serialize(tuple));
    byte[] serialLen = ByteBuffer.allocate(4).putInt(serialData.length).array();


    // Append compressed objects to the file
    FileOutputStream fos = new FileOutputStream(_tupleFileName, true);
    fos.write(serialLen);
    fos.write(serialData);
    fos.close();

  } catch (IOException e) {
    e.printStackTrace();
  }
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public void snappyCompress() {
	byte[] raw = gen_.generate(4 * 1024);
	byte[] compressedOutput = new byte[Snappy
			.maxCompressedLength(raw.length)];

	long produced = 0;

	// attempt to compress the block
	while (bytes_ < 1024 * 1048576) { // Compress 1G
		try {
			int compressedSize = Snappy.compress(raw, 0, raw.length,
					compressedOutput, 0);
			bytes_ += raw.length;
			produced += compressedSize;
		} catch (IOException ignored) {
			throw Throwables.propagate(ignored);
		}

		finishedSingleOp();
	}

	message_ = String.format("(output: %.1f%%)", (produced * 100.0)
			/ bytes_);
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public void snappyUncompressArray() {
	int inputSize = 4 * 1024;
	byte[] compressedOutput = new byte[Snappy
			.maxCompressedLength(inputSize)];
	byte raw[] = gen_.generate(inputSize);
	int compressedLength;
	try {
		compressedLength = Snappy.compress(raw, 0, raw.length,
				compressedOutput, 0);
	} catch (IOException e) {
		throw Throwables.propagate(e);
	}
	// attempt to uncompress the block
	while (bytes_ < 5L * 1024 * 1048576) { // Compress 1G
		try {
			Snappy.uncompress(compressedOutput, 0, compressedLength, raw, 0);
			bytes_ += inputSize;
		} catch (IOException ignored) {
			throw Throwables.propagate(ignored);
		}

		finishedSingleOp();
	}
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
@Override
public int compress(byte[] src, int srcPos, int length, byte[] dest,
	 int destPos) throws IOException {
	System.arraycopy(header, 0, dest, destPos, headerLength);

	// Compressed size cannot be greater than what we have available
	maxCompressedSize = dest.length - destPos - headerLength - 4;
	if (Snappy.maxCompressedLength(length) > maxCompressedSize) {
		return -1;
	}

	compressedLength = Snappy.compress(src, srcPos, length, dest, destPos
		 + headerLength + 4);
	writeInt(compressedLength, dest, destPos + headerLength);

	return headerLength + 4 + compressedLength;
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public byte[] objectToBytes(Object o) throws IOException {
    try {
        byte[] bytes = MarshallerUtil.obj2bytes(marshaller, o);

        if( DEBUG )
            log.debug( StackTraceUtil.getStackTrace(Thread.currentThread().getStackTrace()) );

        if( USE_SNAPPY_COMPRESSION ) {
            byte[] compressBuf = Snappy.compress(bytes);
            if( DEBUG ) {
                log.debug("KhanGridMarshaller/SIZE=" + bytes.length + "/COMPRESS=" + compressBuf.length);
            }
            return compressBuf;
        } else {
            if( DEBUG ) {
                log.debug("KhanGridMarshaller/SIZE=" + bytes.length );
            }
            return bytes;
        }
    } catch (Exception e) {
        throw new IOException("Exception");
    }
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public void compress(ByteBuffer input, ByteBuffer output) throws IOException
{
    int dlimit = output.limit();
    Snappy.compress(input, output);

    // Snappy doesn't match the ICompressor contract w/regards to state it leaves dest ByteBuffer's counters in
    output.position(output.limit());
    output.limit(dlimit);
    input.position(input.limit());
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compressBuffer(ByteBuffer buffer) throws IOException {
    assert(buffer.isDirect());
    IOBuffers buffers = getBuffersForCompression(buffer.remaining(), true);
    ByteBuffer output = buffers.output;

    final int compressedSize = Snappy.compress(buffer, output);
    byte result[] = new byte[compressedSize];
    output.get(result);
    return result;
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compressBytes(byte bytes[], int offset, int length) throws IOException {
    IOBuffers buffers = getBuffersForCompression(bytes.length, false);
    buffers.input.put(bytes, offset, length);
    buffers.input.flip();
    final int compressedSize = Snappy.compress(buffers.input, buffers.output);
    final byte compressed[] = new byte[compressedSize];
    buffers.output.get(compressed);
    return compressed;
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
public static byte[] compressFilter(byte[] datas) {
    if (Constants.COMPRESS_FILE) {
        try {
            datas = Snappy.compress(datas);
        } catch (IOException e) {
            log.error("Compress error!", e);
        }
    }
    return datas;
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
private void writeCompressedBuf(ArrowBuf buf, OutputStream output) throws IOException {
  int rawLength = buf.readableBytes();
  for (int posn = 0; posn < rawLength; posn += RAW_CHUNK_SIZE_TO_COMPRESS) {
    /* we compress 32KB chunks at a time; the last chunk might be smaller than 32KB */
    int lengthToCompress = Math.min(RAW_CHUNK_SIZE_TO_COMPRESS, rawLength - posn);

    /* allocate direct buffers to hold raw and compressed data */
    ByteBuffer rawDirectBuffer = buf.nioBuffer(posn, lengthToCompress);
    /* Since we don't know the exact size of compressed data, we can
     * allocate the compressed buffer of same size as raw data. However,
     * there could be cases where Snappy does not compress the data and the
     * compressed stream is of size larger (raw data + compression metadata)
     * than raw data. To handle these cases, we allocate compressed buffer
     * slightly larger than raw buffer. If we don't do this, Snappy.compress
     * will segfault.
     */
    final int maxCompressedLength = Snappy.maxCompressedLength(lengthToCompress);
    try (ArrowBuf cBuf = allocator.buffer(maxCompressedLength)) {
      ByteBuffer compressedDirectBuffer = cBuf.nioBuffer(0, maxCompressedLength);
      rawDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);
      compressedDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);

      /* compress */
      int compressedLength = Snappy.compress(rawDirectBuffer, compressedDirectBuffer);

      /* get compressed data into byte array for serializing to output stream */
      compressedDirectBuffer.get(tmpBuffer, 0, compressedLength);

      /* serialize the length of compressed data */
      output.write(getByteArrayFromLEInt(compressedLength));
      /* serialize the compressed data */
      output.write(tmpBuffer, 0, compressedLength);
    }
  }
}
 

import org.xerial.snappy.Snappy; //导入方法依赖的package包/类
private static byte[] toString(Serializable obj) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream oos = new ObjectOutputStream(baos);
    oos.writeObject(obj);
    oos.close();
    String stringToSend = Base64.getEncoder().encodeToString(baos.toByteArray());
    byte[] bytesToSend = stringToSend.getBytes();
    return Snappy.compress(bytesToSend);
}