Java SnappyInputStream类代码示例

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
private CatwalkModel fetchModel(NextClientStrategy nextClientStrategy, CatwalkQuery catwalkQuery, String key, int partitionId) throws Exception {

        String json = requestMapper.writeValueAsString(catwalkQuery);
        HttpStreamResponse response = catwalkClient.call("",
            nextClientStrategy,
            "strutModelCacheGet",
            (c) -> new ClientResponse<>(c.streamingPost("/miru/catwalk/model/get/" + key + "/" + partitionId, json, null), true));

        CatwalkModel catwalkModel = null;
        try {
            if (responseMapper.isSuccessStatusCode(response.getStatusCode())) {
                SnappyInputStream in = new SnappyInputStream(new BufferedInputStream(response.getInputStream(), 8192));
                catwalkModel = requestMapper.readValue(in, CatwalkModel.class);
            }
        } finally {
            response.close();
        }

        if (catwalkModel == null) {
            throw new ModelNotAvailable("Model not available,"
                + " status code: " + response.getStatusCode()
                + " reason: " + response.getStatusReasonPhrase());
        }
        return catwalkModel;
    }
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
public static String descompactar(String path) throws IOException {

        String decompressed = path.replaceAll(".cpt", "");

        FileOutputStream fos = new FileOutputStream(decompressed);

        FileInputStream fis = new FileInputStream(path);
        SnappyInputStream siut = new SnappyInputStream(fis);
        BufferedInputStream input = new BufferedInputStream(siut);

        byte[] tmp = new byte[1024];
        for (int readBytes = 0; (readBytes = input.read(tmp)) != -1;) {
            fos.write(tmp, 0, readBytes);
        }
        input.close();
        siut.close();
        fos.flush();
        fos.close();

        return decompressed;

    }
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
public static InputStream toInputStream(com.google.protobuf.ByteString json, FragmentCodec codec) throws IOException {
  final FragmentCodec c = codec != null ? codec : FragmentCodec.NONE;

  final InputStream input = json.newInput();
  switch(c) {
  case NONE:
    return input;

  case SNAPPY:
    return new SnappyInputStream(input);

  default:
    throw new UnsupportedOperationException("Do not know how to uncompress using " + c + " algorithm.");
  }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
/**
 * Deserializes an object from the given array of bytes, e.g., as
 * serialized using {@link #serializeToByteArray}, and returns it.
 *
 * @throws IllegalArgumentException if there are errors when
 * deserializing, using the provided description to identify what
 * was being deserialized
 */
public static Object deserializeFromByteArray(byte[] encodedValue,
    String description) {
  try {
    try (ObjectInputStream ois = new ContextualObjectInputStream(
        new SnappyInputStream(new ByteArrayInputStream(encodedValue)))) {
      return ois.readObject();
    }
  } catch (IOException | ClassNotFoundException exn) {
    throw new IllegalArgumentException(
        "unable to deserialize " + description,
        exn);
  }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public Document fromStream(DataInputStream stream, Document.DocumentComponents selection) throws IOException {
  SerializerCommon.ByteBuf buffer = new SerializerCommon.ByteBuf();
  DataInputStream input = new DataInputStream(new SnappyInputStream(stream));
  Document d = new Document();

  int metadataSize = input.readInt();
  int textSize = input.readInt(); // ignored

  // identifier
  d.identifier = input.readLong();

  // name
  d.name = buffer.readString(input);

  if (selection.metadata) {
    d.metadata = SerializerCommon.readMetadata(input, buffer);
    // only both skipping if we need to
  } else if (selection.text || selection.tokenize) {
    input.skip(metadataSize);
  }

  // can't get tokens without text in this case...
  if (selection.text || selection.tokenize) {
    d.text = SerializerCommon.readText(input, selection, buffer);
  }
  input.close();

  // give back terms & tags
  if(selection.tokenize) {
    // Tokenizer is *not* threadsafe, so we must make a copy of it for each use in case of threads.
    Tokenizer tokenizer = getTokenizer();
    tokenizer.tokenize(d);
  }

  return d;
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public void decompress(InputStream source, OutputStream target) {
    Validate.notNull(source);
    Validate.notNull(target);
    try (InputStream is = new SnappyInputStream(source)) {
        IOUtils.copy(is, target);
    }
    catch (IOException e) {
        throw new IllegalStateException(e);
    } finally {
        IOUtils.closeQuietly(target);
    }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
private List<Datum> readInData() throws IOException {
    File f = new File(fileDir + "/" + convertFileName(timeColumn,
                                                      attributes,
                                                      metrics,
                                                      conf.getString(MacroBaseConf.BASE_QUERY,
                                                                     conf.getString(MacroBaseConf.QUERY_NAME,
                                                                                    "cachedQuery"))));
    if (!f.exists()) {
        log.info("Data did not exist; going to read from SQL.");
        return null;
    }

    log.info("On-disk cache exists; loading...");
    InputStream inputStream = new SnappyInputStream(new BufferedInputStream(new FileInputStream(f), 16384));

    Kryo kryo = new Kryo();
    Input input = new Input(inputStream);

    DatumEncoder cachedEncoder = kryo.readObject(input, DatumEncoder.class);

    Integer numBatches = kryo.readObject(input, Integer.class);
    List<Datum> output = null;

    for (int i = 0; i < numBatches; ++i) {
        List<Datum> fromDisk = (List<Datum>) kryo.readClassAndObject(input);
        if (output == null) {
            output = fromDisk;
        } else {
            output.addAll(fromDisk);
        }
    }

    log.info("...loaded!");

    conf.getEncoder().copy(cachedEncoder);
    return output;
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
private void handleModernVersion(int version, int header) throws IOException
{
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    out.writeInt(MessagingService.current_version);
    out.flush();

    DataInputStream in = new DataInputStream(socket.getInputStream());
    int maxVersion = in.readInt();
    from = CompactEndpointSerializationHelper.deserialize(in);
    boolean compressed = MessagingService.getBits(header, 2, 1) == 1;

    if (compressed)
    {
        logger.debug("Upgrading incoming connection to be compressed");
        in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
    }
    else
    {
        in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), 4096));
    }

    logger.debug("Max version for {} is {}", from, maxVersion);
    if (version > MessagingService.current_version)
    {
        // save the endpoint so gossip will reconnect to it
        Gossiper.instance.addSavedEndpoint(from);
        logger.info("Received messages from newer protocol version {}. Ignoring", version);
        return;
    }
    MessagingService.instance().setVersion(from, Math.min(MessagingService.current_version, maxVersion));
    logger.debug("set version for {} to {}", from, Math.min(MessagingService.current_version, maxVersion));
    // outbound side will reconnect if necessary to upgrade version

    while (true)
    {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
private void handleModernVersion(int version, int header) throws IOException
{
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    out.writeInt(MessagingService.current_version);
    out.flush();

    DataInputStream in = new DataInputStream(socket.getInputStream());
    int maxVersion = in.readInt();
    from = CompactEndpointSerializationHelper.deserialize(in);
    boolean compressed = MessagingService.getBits(header, 2, 1) == 1;

    if (compressed)
    {
        logger.debug("Upgrading incoming connection to be compressed");
        in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
    }
    else
    {
        in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), 4096));
    }

    logger.debug("Max version for {} is {}", from, maxVersion);
    if (version > MessagingService.current_version)
    {
        // save the endpoint so gossip will reconnect to it
        Gossiper.instance.addSavedEndpoint(from);
        logger.info("Received messages from newer protocol version {}. Ignoring", version);
        return;
    }
    MessagingService.instance().setVersion(from, maxVersion);
    logger.debug("Set version for {} to {} (will use {})", from, maxVersion, Math.min(MessagingService.current_version, maxVersion));
    // outbound side will reconnect if necessary to upgrade version

    while (true)
    {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
public static InputStream create(final CompressionCodec compressionCodec, final InputStream stream) throws IOException {
    switch (compressionCodec) {
        case NONE:
            return stream;
        case GZIP:
            return new GZIPInputStream(stream);
        case SNAPPY:
            return new SnappyInputStream(stream);
        case LZ4:
            // TODO Implement KafkaLZ4BlockInputStream
            //return new new KafkaLZ4BlockInputStream(stream);
        default:
            throw new UnknownCodecException("Unknown Codec: " + compressionCodec);
    }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
public static InputStream apply(CompressionCodec compressionCodec, InputStream stream) {
    try {
        if (compressionCodec == DefaultCompressionCodec.instance) return new GZIPInputStream(stream);
        if (compressionCodec == GZIPCompressionCodec.instance) return new GZIPInputStream(stream);
        if (compressionCodec == SnappyCompressionCodec.instance) return new SnappyInputStream(stream);
    } catch (IOException e) {
        throw new KafkaException(e);
    }
    throw new kafka.common.UnknownCodecException("Unknown Codec: " + compressionCodec);
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
/**
 * Decompresses input files that were snappy compressed before opening them with the sstable
 * reader. It writes a new decompressed file with the same name as the compressed one. The old
 * one gets deleted.
 */
private void decompress(Path localTablePath, TaskAttemptContext context) throws IOException {
    context.setStatus(String.format("Decompressing %s", localTablePath.toUri()));
    int compressionBufSize = context.getConfiguration().getInt(
            PropertyConstants.DECOMPRESS_BUFFER.txt, DEFAULT_DECOMPRESS_BUFFER_SIZE);
    compressionBufSize *= 1024;
    LOG.info("Decompressing {} with buffer size {}.", localTablePath, compressionBufSize);
    File compressedFile = new File(localTablePath.toString());
    InputStream fis = new FileInputStream(compressedFile);
    InputStream bis = new BufferedInputStream(fis, compressionBufSize);
    InputStream sip = new SnappyInputStream(bis);
    File decompressedFile = new File(localTablePath.toString() + ".tmp");

    OutputStream os = new FileOutputStream(decompressedFile);
    OutputStream bos = new BufferedOutputStream(os, compressionBufSize);
    byte[] inByteArr = new byte[compressionBufSize];
    int bytesRead = 0;
    int bytesSinceLastReport = 0;
    while ((bytesRead = sip.read(inByteArr)) > 0) {
        bos.write(inByteArr, 0, bytesRead);
        bytesSinceLastReport += bytesRead;
        // Avoid timeouts. Report progress to the jobtracker.
        if (bytesSinceLastReport % REPORT_DECOMPRESS_PROGRESS_EVERY_GBS > 0) {
            context.setStatus(String.format("Decompressed %d bytes.", bytesSinceLastReport));
            bytesSinceLastReport -= REPORT_DECOMPRESS_PROGRESS_EVERY_GBS;
        }
    }
    sip.close();
    bos.close();
    compressedFile.delete();
    decompressedFile.renameTo(compressedFile);
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public void availableRowsStream(RingMember localRingMember,
    TimestampedRingHost localTimestampedRingHost,
    RingMember remoteRingMember,
    RingHost remoteRingHost,
    boolean system,
    long takeSessionId,
    long takeSharedKey,
    long timeoutMillis,
    AvailableStream availableStream,
    PingStream pingStream) throws Exception {

    String endpoint = "/amza/rows/available"
        + "/" + localRingMember.getMember()
        + "/" + localTimestampedRingHost.ringHost.toCanonicalString()
        + "/" + system
        + "/" + localTimestampedRingHost.timestampId
        + "/" + takeSessionId
        + "/" + timeoutMillis;
    String sharedKeyJson = mapper.writeValueAsString(takeSharedKey); // lame
    HttpStreamResponse httpStreamResponse = ringClient.call("",
        new ConnectionDescriptorSelectiveStrategy(new HostPort[] { new HostPort(remoteRingHost.getHost(), remoteRingHost.getPort()) }),
        "availableRowsStream",
        httpClient -> {
            HttpStreamResponse response = httpClient.streamingPost(endpoint, sharedKeyJson, null);
            if (response.getStatusCode() < 200 || response.getStatusCode() >= 300) {
                throw new NonSuccessStatusCodeException(response.getStatusCode(), response.getStatusReasonPhrase());
            }
            return new ClientResponse<>(response, true);
        });
    try {
        BufferedInputStream bis = new BufferedInputStream(httpStreamResponse.getInputStream(), 8192); // TODO config??
        DataInputStream dis = new DataInputStream(new SnappyInputStream(bis));
        streamingTakesConsumer.consume(dis, availableStream, pingStream);
    } finally {
        httpStreamResponse.close();
    }
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void testCreateInputStream() throws IOException {
  SnappyCompressionProvider provider =
    (SnappyCompressionProvider) factory.getCompressionProviderByName( PROVIDER_NAME );
  SnappyInputStream in = createSnappyInputStream();
  SnappyCompressionInputStream inStream = new SnappyCompressionInputStream( in, provider );
  assertNotNull( inStream );
  SnappyCompressionInputStream ncis = provider.createInputStream( in );
  assertNotNull( ncis );
}
 

import org.xerial.snappy.SnappyInputStream; //导入依赖的package包/类
private SnappyInputStream createSnappyInputStream() throws IOException {
  // Create an in-memory ZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  SnappyOutputStream sos = new SnappyOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  sos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  sos.close();

  return new SnappyInputStream( in );
}