Java SnappyInputStream类代码示例

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void allFloatParameterCombinationsKnockout() throws IOException {
    final List<float[]> fullData = loadFullFloatData();

    try (BufferedWriter noSplitWriter = openWriter("floats-knockout-nosplit");
         BufferedWriter splitWriter = openWriter("floats-knockout")) {
        for (float knockout : new float[] { 0f, 0.1f, 0.5f, 0.75f }) {
            final Random r = new Random(1337);
            final List<float[]> check = fullData.stream().map(xs -> {
                final float[] ys = xs.clone();
                for (int i = 0; i < ys.length; i++) {
                    if (r.nextDouble() < knockout) {
                        ys[i] = Float.NaN;
                    }
                }
                return ys;
            }).collect(Collectors.toList());

            allFloatParameterCombinationsWork(check, "Snappy\t" + knockout, SnappyOutputStream::new, SnappyInputStream::new, noSplitWriter, splitWriter);
        }
    }
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void allDoubleParameterCombinationsKnockout() throws IOException {
    final List<double[]> fullData = loadFullDoubleData();

    try (BufferedWriter noSplitWriter = openWriter("doubles-knockout-nosplit");
         BufferedWriter splitWriter = openWriter("doubles-knockout")) {
        for (float knockout : new float[] { 0f, 0.1f, 0.5f, 0.75f }) {
            final Random r = new Random(1337);
            final List<double[]> check = fullData.stream().map(xs -> {
                final double[] ys = xs.clone();
                for (int i = 0; i < ys.length; i++) {
                    if (r.nextDouble() < knockout) {
                        ys[i] = Double.NaN;
                    }
                }
                return ys;
            }).collect(Collectors.toList());

            allDoubleParameterCombinationsWork(check, "Snappy\t" + knockout, SnappyOutputStream::new, SnappyInputStream::new, noSplitWriter, splitWriter);
        }
    }
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public void init(File file) {
	// currentFile = file;
	try {
		is = new DataInputStream(new SnappyInputStream(
				new BufferedInputStream(new FileInputStream(file),
						65536)));
		signature[0] = triple[0];
		signature[1] = triple[1];
		signature[2] = triple[2];
		signature[3] = step;
		signature[4] = count;
	} catch (Exception e) {
		log.error("Error", e);
	}
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void benchmark() throws IOException {
    /*
    Snappy compress: 0.156672s
    Compressed to 10876881 bytes
    Uncompresses to 12908620 bytes
    Snappy decompress: 0.162463s

    Deflate Fastest compress: 6.017877s
    Compressed to 7770401 bytes
    Uncompresses to 12908620 bytes
    Deflate Fastest decompress: 4.260146s

    Deflate Slowest compress: 7.06043s
    Compressed to 7492001 bytes
    Uncompresses to 12908620 bytes
    Deflate Slowest decompress: 3.968221s

    Deflate Normal compress: 5.201332s
    Compressed to 7515522 bytes
    Uncompresses to 12908620 bytes
    Deflate Normal decompress: 4.056792s
     */
    benchmark("None",            x -> x,                                                                         x -> x);
    benchmark("None",            x -> x,                                                                         x -> x);
    benchmark("Snappy",          SnappyOutputStream::new,                                                        SnappyInputStream::new);
    benchmark("GZip",            GZIPOutputStream::new,                                                          GZIPInputStream::new);
    benchmark("Deflate Fastest", os -> new DeflaterOutputStream(os, new Deflater(Deflater.BEST_SPEED)),          InflaterInputStream::new);
    benchmark("Deflate Normal",  os -> new DeflaterOutputStream(os, new Deflater(Deflater.DEFAULT_COMPRESSION)), InflaterInputStream::new);
    benchmark("Deflate Slowest", os -> new DeflaterOutputStream(os, new Deflater(Deflater.BEST_COMPRESSION)),    InflaterInputStream::new);
    benchmark("BZip2 Fastest",   os -> new BZip2CompressorOutputStream(os, BZip2CompressorOutputStream.MIN_BLOCKSIZE), BZip2CompressorInputStream::new);
    benchmark("BZip2 Slowest",   os -> new BZip2CompressorOutputStream(os, BZip2CompressorOutputStream.MAX_BLOCKSIZE), BZip2CompressorInputStream::new);
    benchmark("XZ Fastest",      os -> new XZCompressorOutputStream(os, LZMA2Options.PRESET_MIN),                      XZCompressorInputStream::new);
    benchmark("XZ Normal",       os -> new XZCompressorOutputStream(os, LZMA2Options.PRESET_DEFAULT),                  XZCompressorInputStream::new);
    benchmark("XZ Slowest",      os -> new XZCompressorOutputStream(os, LZMA2Options.PRESET_MAX),                      XZCompressorInputStream::new);
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void allFloatParameterCombinationsWork() throws IOException {
    try (BufferedWriter noSplitWriter = openWriter("floats-nosplit");
         BufferedWriter splitWriter = openWriter("floats")) {
        allFloatParameterCombinationsWork(loadFullFloatData(), "Snappy", SnappyOutputStream::new,                                                              SnappyInputStream::new,          noSplitWriter, splitWriter);
        allFloatParameterCombinationsWork(loadFullFloatData(), "BZ2",    os -> new BZip2CompressorOutputStream(os, BZip2CompressorOutputStream.MIN_BLOCKSIZE), BZip2CompressorInputStream::new, noSplitWriter, splitWriter);
    }
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Test
public void allDoubleParameterCombinationsWork() throws IOException {
    try (BufferedWriter noSplitWriter = openWriter("doubles-nosplit");
         BufferedWriter splitWriter = openWriter("doubles")) {
        allDoubleParameterCombinationsWork(loadFullDoubleData(), "Snappy", SnappyOutputStream::new,                                                              SnappyInputStream::new,          noSplitWriter, splitWriter);
        allDoubleParameterCombinationsWork(loadFullDoubleData(), "BZ2",    os -> new BZip2CompressorOutputStream(os, BZip2CompressorOutputStream.MIN_BLOCKSIZE), BZip2CompressorInputStream::new, noSplitWriter, splitWriter);
    }
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
ListBlocks readListBlockFromFile(int no) throws Exception {
	String dir = index.cacheDir;
	String name = "lb-" + no;
	if (!listBlockNameSet.contains(name)) {
		return null;
	}
	getIOLock();
	File file = index.getCacheFile(name + "/index");
	try {
		InputStream reader = index.getFilesInterface().createInputStream(
				file);
		DataInputStream stream = new DataInputStream(new SnappyInputStream(
				reader));
		// reader);
		ListBlocks block = new ListBlocks(no, name, 1);
		block.readFrom(stream);
		stream.close();
		reader.close();
		if (log.isDebugEnabled()) {
			log.debug("done reading index of ListBlock " + dir + "/" + name);
		}
		return block;
	} catch (Exception e) {
		log.error("Failed reading the file: " + file, e);
		throw e;
	} finally {
		releaseIOLock();
	}
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public void openStream() throws IOException {
	if (stream == null) {
		stream = new FDataInput(new SnappyInputStream(
				new BufferedInputStream(new FileInputStream(filename))));
	}
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
public InputStream(java.io.InputStream stream) throws IOException {
  myUnderlyingInputStream = new SnappyInputStream(stream);
  myInputStream = new CompactDataInput(myUnderlyingInputStream);
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Benchmark
public int snappy() throws IOException {
    return benchmark(SnappyOutputStream::new,                                                        SnappyInputStream::new);
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
@Override
public InputStream input(InputStream in) throws IOException {
    return new SnappyInputStream(in);
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
int[] readBlock(String name, int blockNo, ActionContext context)
		throws Exception {
	getIOLock();
	File f = index.getCacheFile(name + "/" + blockNo);
	int[] result = null;
	try {
		if (log.isDebugEnabled()) {
			log.debug("Reading block " + blockNo + " of " + name);
		}
		long time = System.currentTimeMillis();
		InputStream reader = index.getFilesInterface().createInputStream(f);
		DataInputStream stream = new DataInputStream(new SnappyInputStream(
				reader));
		// reader);
		int sz = stream.readInt();
		result = new int[sz];
		result[0] = sz;
		synchronized (tmpBuffer) {
			if (tmpBuffer.length < (sz - 1) * 4) {
				tmpBuffer = new byte[(sz - 1) * 4];
				intBuffer = ByteBuffer.wrap(tmpBuffer).asIntBuffer();
			}

			stream.readFully(tmpBuffer, 0, (sz - 1) * 4);
			intBuffer.rewind();
			intBuffer.get(result, 1, sz - 1);
		}
		stream.close();
		reader.close();

		/*
		 * if (me == -1) { me = context.getNetworkLayer().getMyPartition();
		 * } context.incrCounter("Node " + me +
		 * ", time spent reading lb from disk", System.currentTimeMillis() -
		 * time); context.incrCounter("Node " + me +
		 * ", lb bytes read from disk", sz * 4);
		 */
		if (context != null) {
			context.incrCounter("Time spent reading lb from disk",
					System.currentTimeMillis() - time);
			context.incrCounter("Lb bytes read from disk", sz * 4);
		}
		totalListBlocksSize += sz;
		if (totalListBlocksSize > MAX_LISTBLOCKSSIZE_TO_KEEP_CACHE) {
			// We already have the maximum number of
			// ListBlocks in core, so we have
			// to delete one.
			removeOldestBlockInCache();
		}
		if (log.isDebugEnabled()) {
			log.debug("After readBlock: totalListBlocksSize = "
					+ totalListBlocksSize);
		}
	} catch (IOException e) {
		log.error("Could not read block " + f.getAbsolutePath(), e);
	} finally {
		releaseIOLock();
	}
	return result;
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
private Block getBlock(int i, boolean removeOldest, ActionContext context) {
	synchronized (blocksCache) {
		Block block = blocksCache.get(myId + i);
		if (block == null) {
			FirstLayer.getIOLock();
			File f = null;
			try {
				long time = System.currentTimeMillis();
				f = getCacheFile("" + (i / 1000) + "/" + i);
				if (log.isDebugEnabled()) {
					log.debug("Reading block " + f.getPath());
				}
				InputStream fin = fi.createInputStream(f);
				InputStream stream = new SnappyInputStream(fin);
				DataInputStream din = new DataInputStream(stream);
				// DataInputStream din = new DataInputStream(fin);
				block = new Block();
				int sz = din.readInt();
				block.block = new byte[sz];
				block.index = myId + i;
				blocksCache.addBlock(block);
				din.readFully(block.block);
				din.close();
				fin.close();
				if (context != null) {
					/*
					 * if (me == -1) { me =
					 * context.getNetworkLayer().getMyPartition(); }
					 * context.incrCounter( "Node " + me +
					 * ", time spent reading block from disk",
					 * System.currentTimeMillis() - time);
					 * context.incrCounter("Node " + me +
					 * ", bytes read from disk", sz+4);
					 */
					context.incrCounter(
							"Time spent reading block from disk",
							System.currentTimeMillis() - time);
					context.incrCounter("Bytes read from disk", sz + 4);
				}
			} catch (Exception e) {
				log.error("Failed reading cache file " + f.getPath(), e);
				throw new Error("Failed reading cache file " + f.getPath(),
						e);
			} finally {
				FirstLayer.releaseIOLock();
			}

			if (removeOldest
					&& blocksCache.size() >= MAX_BUFFERS_TO_KEEP_CACHE) {
				Block b = blocksCache.removeOldestBlockInCache();
				if (b.marked) {
					writeBlockToDisk(b);
					b.marked = false;
				}
			}
		} else {
			blocksCache.moveToEnd(block);
		}

		return block;
	}
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
private void consistencyCheck() {
	try {
		stream = new FDataInput(new SnappyInputStream(
				new BufferedInputStream(new FileInputStream(filename))));
		long nel = nElements - 1;
		long sz = size;
		if (nel > 0) {
			if (sz <= 0) {
				log.error("inconsistency in file " + filename,
						new Throwable());
				throw new Error("Inconsistency");
			}
		}
		if (nel == 0 && sz != 0) {
			log.error("inconsistency in file " + filename, new Throwable());
			throw new Error("Inconsistency");
		}
		while (nel > 0) {
			// Consistency check
			int length = stream.readInt();
			if (length != current.length) {
				current = new byte[length];
			}
			if (length < 0 || length > 4096) {
				log.error("inconsistency in file " + filename,
						new Throwable());
				throw new Error("Inconsistency");
			}
			stream.readFully(current);
			sz -= 4 + length;
			nel--;
		}
		if (sz != 0) {
			log.error("inconsistency in file " + filename, new Throwable());
			throw new Error("Inconsistency");
		}
		stream.close();
	} catch (Throwable e) {
		log.error("Got exception in consistency check", e);
		throw new Error("Inconsistency", e);
	}
	stream = null;
}
 

import org.iq80.snappy.SnappyInputStream; //导入依赖的package包/类
protected InputStream wrapInputStream(InputStream underlying) throws IOException {
    return new SnappyInputStream(underlying);
}