Java Bzip2Factory类代码示例

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
@Test(timeout=20000)
public void testBZip2NativeCodec() throws IOException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
      conf.set("io.compression.codec.bzip2.library", "java-builtin");
      codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec");
      codecTest(conf, seed, count, 
                "org.apache.hadoop.io.compress.BZip2Codec");
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
@Test(timeout=20000)
public void testSequenceFileBZip2NativeCodec() throws IOException, 
                      ClassNotFoundException, InstantiationException, 
                      IllegalAccessException {
  Configuration conf = new Configuration();
  conf.set("io.compression.codec.bzip2.library", "system-native");
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    if (Bzip2Factory.isNativeBzip2Loaded(conf)) {
      sequenceFileCodecTest(conf, 0, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 100, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 100);
      sequenceFileCodecTest(conf, 200000, 
                            "org.apache.hadoop.io.compress.BZip2Codec", 
                            1000000);
    } else {
      LOG.warn("Native hadoop library available but native bzip2 is not");
    }
  }
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
@Test
public void testNative()
{
    HadoopNative.requireHadoopNative();

    assertTrue(NativeCodeLoader.isNativeCodeLoaded());
    assertTrue(NativeCodeLoader.buildSupportsSnappy());
    assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration()));
    assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration()));
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
/**
 * A tool to test native library availability, 
 */
public static void main(String[] args) {
  String usage = "NativeLibraryChecker [-a|-h]\n"
      + "  -a  use -a to check all libraries are available\n"
      + "      by default just check hadoop library is available\n"
      + "      exit with error code 1 if check failed\n"
      + "  -h  print this message\n";
  if (args.length > 1 ||
      (args.length == 1 &&
          !(args[0].equals("-a") || args[0].equals("-h")))) {
    System.err.println(usage);
    ExitUtil.terminate(1);
  }
  boolean checkAll = false;
  if (args.length == 1) {
    if (args[0].equals("-h")) {
      System.out.println(usage);
      return;
    }
    checkAll = true;
  }
  Configuration conf = new Configuration();
  boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
  boolean zlibLoaded = false;
  boolean snappyLoaded = false;
  // lz4 is linked within libhadoop
  boolean lz4Loaded = nativeHadoopLoaded;
  boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
  String hadoopLibraryName = "";
  String zlibLibraryName = "";
  String snappyLibraryName = "";
  String lz4LibraryName = "";
  String bzip2LibraryName = "";
  if (nativeHadoopLoaded) {
    hadoopLibraryName = NativeCodeLoader.getLibraryName();
    zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
    if (zlibLoaded) {
      zlibLibraryName = ZlibFactory.getLibraryName();
    }
    snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
        SnappyCodec.isNativeCodeLoaded();
    if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
      snappyLibraryName = SnappyCodec.getLibraryName();
    }
    if (lz4Loaded) {
      lz4LibraryName = Lz4Codec.getLibraryName();
    }
    if (bzip2Loaded) {
      bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
    }
  }
  System.out.println("Native library checking:");
  System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
  System.out.printf("zlib:   %b %s\n", zlibLoaded, zlibLibraryName);
  System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
  System.out.printf("lz4:    %b %s\n", lz4Loaded, lz4LibraryName);
  System.out.printf("bzip2:  %b %s\n", bzip2Loaded, bzip2LibraryName);
  if ((!nativeHadoopLoaded) ||
      (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
    // return 1 to indicated check failed
    ExitUtil.terminate(1);
  }
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
/**
 * Create a {@link CompressionOutputStream} that will write to the given
 * {@link OutputStream} with the given {@link Compressor}.
 *
 * @param out        the location for the final output stream
 * @param compressor compressor to use
 * @return a stream the user can write uncompressed data to, to have it 
 *         compressed
 * @throws IOException
 */
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
    Compressor compressor) throws IOException {
  return Bzip2Factory.isNativeBzip2Loaded(conf) ?
    new CompressorStream(out, compressor, 
                         conf.getInt("io.file.buffer.size", 4*1024)) :
    new BZip2CompressionOutputStream(out);
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
/**
 * Create a {@link CompressionInputStream} that will read from the given
 * {@link InputStream} with the given {@link Decompressor}, and return a 
 * stream for uncompressed data.
 *
 * @param in           the stream to read compressed bytes from
 * @param decompressor decompressor to use
 * @return a stream to read uncompressed bytes from
 * @throws IOException
 */
@Override
public CompressionInputStream createInputStream(InputStream in,
    Decompressor decompressor) throws IOException {
  return Bzip2Factory.isNativeBzip2Loaded(conf) ? 
    new DecompressorStream(in, decompressor,
                           conf.getInt("io.file.buffer.size", 4*1024)) :
    new BZip2CompressionInputStream(in);
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
/**
 * Create a {@link CompressionOutputStream} that will write to the given
 * {@link OutputStream} with the given {@link Compressor}.
 *
 * @param out        the location for the final output stream
 * @param compressor compressor to use
 * @return a stream the user can write uncompressed data to, to have it 
 *         compressed
 * @throws IOException
 */
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
    Compressor compressor) throws IOException {
  return Bzip2Factory.isNativeBzip2Loaded(conf) ?
    new CompressorStream(out, compressor, 
                         conf.getInt(IO_FILE_BUFFER_SIZE_KEY,
                                 IO_FILE_BUFFER_SIZE_DEFAULT)) :
    new BZip2CompressionOutputStream(out);
}
 

import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; //导入依赖的package包/类
/**
 * Create a {@link CompressionInputStream} that will read from the given
 * {@link InputStream} with the given {@link Decompressor}, and return a 
 * stream for uncompressed data.
 *
 * @param in           the stream to read compressed bytes from
 * @param decompressor decompressor to use
 * @return a stream to read uncompressed bytes from
 * @throws IOException
 */
@Override
public CompressionInputStream createInputStream(InputStream in,
    Decompressor decompressor) throws IOException {
  return Bzip2Factory.isNativeBzip2Loaded(conf) ? 
    new DecompressorStream(in, decompressor,
                           conf.getInt(IO_FILE_BUFFER_SIZE_KEY,
                               IO_FILE_BUFFER_SIZE_DEFAULT)) :
    new BZip2CompressionInputStream(in);
}