Java ZlibFactory.setNativeZlibLoaded方法代码示例

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testCodecInitWithCompressionLevel() throws Exception {
  Configuration conf = new Configuration();
  if (ZlibFactory.isNativeZlibLoaded(conf)) {
    LOG.info("testCodecInitWithCompressionLevel with native");
    codecTestWithNOCompression(conf,
                          "org.apache.hadoop.io.compress.GzipCodec");
    codecTestWithNOCompression(conf,
                       "org.apache.hadoop.io.compress.DefaultCodec");
  } else {
    LOG.warn("testCodecInitWithCompressionLevel for native skipped"
             + ": native libs not loaded");
  }
  conf = new Configuration();
  // don't use native libs
  ZlibFactory.setNativeZlibLoaded(false);
  codecTestWithNOCompression( conf,
                       "org.apache.hadoop.io.compress.DefaultCodec");
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testCodecPoolCompressorReinit() throws Exception {
  Configuration conf = new Configuration();
  if (ZlibFactory.isNativeZlibLoaded(conf)) {
    GzipCodec gzc = ReflectionUtils.newInstance(GzipCodec.class, conf);
    gzipReinitTest(conf, gzc);
  } else {
    LOG.warn("testCodecPoolCompressorReinit skipped: native libs not loaded");
  }
  // don't use native libs
  ZlibFactory.setNativeZlibLoaded(false);
  DefaultCodec dfc = ReflectionUtils.newInstance(DefaultCodec.class, conf);
  gzipReinitTest(conf, dfc);
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testGzipCompatibility() throws IOException {
  Random r = new Random();
  long seed = r.nextLong();
  r.setSeed(seed);
  LOG.info("seed: " + seed);

  DataOutputBuffer dflbuf = new DataOutputBuffer();
  GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
  byte[] b = new byte[r.nextInt(128 * 1024 + 1)];
  r.nextBytes(b);
  gzout.write(b);
  gzout.close();

  DataInputBuffer gzbuf = new DataInputBuffer();
  gzbuf.reset(dflbuf.getData(), dflbuf.getLength());

  Configuration conf = new Configuration();
  // don't use native libs
  ZlibFactory.setNativeZlibLoaded(false);
  CompressionCodec codec = ReflectionUtils.newInstance(GzipCodec.class, conf);
  Decompressor decom = codec.createDecompressor();
  assertNotNull(decom);
  assertEquals(BuiltInGzipDecompressor.class, decom.getClass());
  InputStream gzin = codec.createInputStream(gzbuf, decom);

  dflbuf.reset();
  IOUtils.copyBytes(gzin, dflbuf, 4096);
  final byte[] dflchk = Arrays.copyOf(dflbuf.getData(), dflbuf.getLength());
  assertArrayEquals(b, dflchk);
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testBuiltInGzipConcat() throws IOException {
  Configuration conf = new Configuration();
  // don't use native libs
  ZlibFactory.setNativeZlibLoaded(false);
  GzipConcatTest(conf, BuiltInGzipDecompressor.class);
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testGzipCodecRead() throws IOException {
  // Create a gzipped file and try to read it back, using a decompressor
  // from the CodecPool.

  // Don't use native libs for this test.
  Configuration conf = new Configuration();
  ZlibFactory.setNativeZlibLoaded(false);
  // Ensure that the CodecPool has a BuiltInZlibInflater in it.
  Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!", zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",
      zlibDecompressor instanceof BuiltInZlibInflater);
  CodecPool.returnDecompressor(zlibDecompressor);

  // Now create a GZip text file.
  String tmpDir = System.getProperty("test.build.data", "/tmp/");
  Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
  BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
    new GZIPOutputStream(new FileOutputStream(f.toString()))));
  final String msg = "This is the message in the file!";
  bw.write(msg);
  bw.close();

  // Now read it back, using the CodecPool to establish the
  // decompressor to use.
  CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
  CompressionCodec codec = ccf.getCodec(f);
  Decompressor decompressor = CodecPool.getDecompressor(codec);
  FileSystem fs = FileSystem.getLocal(conf);
  InputStream is = fs.open(f);
  is = codec.createInputStream(is, decompressor);
  BufferedReader br = new BufferedReader(new InputStreamReader(is));
  String line = br.readLine();
  assertEquals("Didn't get the same message back!", msg, line);
  br.close();
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testGzipLongOverflow() throws IOException {
  LOG.info("testGzipLongOverflow");

  // Don't use native libs for this test.
  Configuration conf = new Configuration();
  ZlibFactory.setNativeZlibLoaded(false);
  assertFalse("ZlibFactory is using native libs against request",
      ZlibFactory.isNativeZlibLoaded(conf));

  // Ensure that the CodecPool has a BuiltInZlibInflater in it.
  Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!", zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",
      zlibDecompressor instanceof BuiltInZlibInflater);
  CodecPool.returnDecompressor(zlibDecompressor);

  // Now create a GZip text file.
  String tmpDir = System.getProperty("test.build.data", "/tmp/");
  Path f = new Path(new Path(tmpDir), "testGzipLongOverflow.bin.gz");
  BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
    new GZIPOutputStream(new FileOutputStream(f.toString()))));

  final int NBUF = 1024 * 4 + 1;
  final char[] buf = new char[1024 * 1024];
  for (int i = 0; i < buf.length; i++) buf[i] = '\0';
  for (int i = 0; i < NBUF; i++) {
    bw.write(buf);
  }
  bw.close();

  // Now read it back, using the CodecPool to establish the
  // decompressor to use.
  CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
  CompressionCodec codec = ccf.getCodec(f);
  Decompressor decompressor = CodecPool.getDecompressor(codec);
  FileSystem fs = FileSystem.getLocal(conf);
  InputStream is = fs.open(f);
  is = codec.createInputStream(is, decompressor);
  BufferedReader br = new BufferedReader(new InputStreamReader(is));
  for (int j = 0; j < NBUF; j++) {
    int n = br.read(buf);
    assertEquals("got wrong read length!", n, buf.length);
    for (int i = 0; i < buf.length; i++)
      assertEquals("got wrong byte!", buf[i], '\0');
  }
  br.close();
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testGzipCodecWriteJava() throws IOException {
  // don't use native libs
  ZlibFactory.setNativeZlibLoaded(false);
  testGzipCodecWrite(false);
}
 

import org.apache.hadoop.io.compress.zlib.ZlibFactory; //导入方法依赖的package包/类
@Test
public void testCodecPoolAndGzipDecompressor() {
  // BuiltInZlibInflater should not be used as the GzipCodec decompressor.
  // Assert that this is the case.

  // Don't use native libs for this test.
  Configuration conf = new Configuration();
  ZlibFactory.setNativeZlibLoaded(false);
  assertFalse("ZlibFactory is using native libs against request",
              ZlibFactory.isNativeZlibLoaded(conf));

  // This should give us a BuiltInZlibInflater.
  Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!", zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",
      zlibDecompressor instanceof BuiltInZlibInflater);
  // its createOutputStream() just wraps the existing stream in a
  // java.util.zip.GZIPOutputStream.
  CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
  CompressionCodec codec = ccf.getCodec(new Path("foo.gz"));
  assertTrue("Codec for .gz file is not GzipCodec", 
      codec instanceof GzipCodec);

  // make sure we don't get a null decompressor
  Decompressor codecDecompressor = codec.createDecompressor();
  if (null == codecDecompressor) {
    fail("Got null codecDecompressor");
  }

  // Asking the CodecPool for a decompressor for GzipCodec
  // should not return null
  Decompressor poolDecompressor = CodecPool.getDecompressor(codec);
  if (null == poolDecompressor) {
    fail("Got null poolDecompressor");
  }
  // return a couple decompressors
  CodecPool.returnDecompressor(zlibDecompressor);
  CodecPool.returnDecompressor(poolDecompressor);
  Decompressor poolDecompressor2 = CodecPool.getDecompressor(codec);
  if (poolDecompressor.getClass() == BuiltInGzipDecompressor.class) {
    if (poolDecompressor == poolDecompressor2) {
      fail("Reused java gzip decompressor in pool");
    }
  } else {
    if (poolDecompressor != poolDecompressor2) {
      fail("Did not reuse native gzip decompressor in pool");
    }
  }
}