Java CompressorStreamFactory類代碼示例

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
private static int getTriplesCount(String uri, String compression, RDFFormat format) throws Exception {
    InputStream in = FileSystem.get(URI.create(uri), HBaseServerTestInstance.getInstanceConfig()).open(new Path(uri));
    try {
        if (compression != null) {
            in = new CompressorStreamFactory().createCompressorInputStream(compression, in);
        }
        RDFParser parser = Rio.createParser(format);
        final AtomicInteger i = new AtomicInteger();
        parser.setRDFHandler(new AbstractRDFHandler(){
            @Override
            public void handleStatement(Statement st) throws RDFHandlerException {
                i.incrementAndGet();
            }
        });
        parser.parse(in, uri);
        return i.get();
    } finally {
        in.close();
    }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
public static List<A2Encoder> getEncoders() {
	List<A2Encoder> list = new ArrayList<>();
	list.add(new RleEncoder());
	list.add(new VariableRleEncoder());
	list.add(new PackBitsEncoder());
	list.add(new BitPack1());
	list.add(new BitPack2());
	list.add(new BitPack3());
	list.add(new GZipEncoder());
	list.add(new ZipEncoder());
	// From Apache Commons
	for (String outputProvider : CompressorStreamFactory.findAvailableCompressorOutputStreamProviders().keySet()) {
		// PACK200 does nothing for some reason, so just ignoring it
		if (CompressorStreamFactory.PACK200.equals(outputProvider)) continue;
		list.add(new CommonsCodecEncoder(outputProvider));
	}
	return list;
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
public XmlTraceStreamWriter(File file) throws TurnusException {
	try {
		String extension = FileUtils.getExtension(file);
		compressedXml = false;
		if (extension.equals(TRACEZ)) {
			compressedXml = true;
		} else if (!extension.equals(TRACE)) {
			throw new IOException("Trace file writer: unsupported file extension");
		}

		stream = new FileOutputStream(file);
		if (compressedXml) {
			stream = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP,
					stream);
		}
		stream = new BufferedOutputStream(stream, DEFAULT_STREAM_BUFFER_SIZE);

	} catch (Exception e) {
		throw new TurnusException("Trace file writer cannot be create", e);
	}
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
CompressorInputStream createCompressorInputStream(String format,
        InputStream in) throws IOException, CompressorException {
    CompressorStreamFactory factory = new CompressorStreamFactory();
    factory.setDecompressConcatenated(decompressConcatenated);
    if (CommonsCompressUtil.isAutoDetect(format)) {
        in = in.markSupported() ? in : new BufferedInputStream(in);
        try {
            return factory.createCompressorInputStream(in);
        } catch (CompressorException e) {
            throw new IOException(
                    "Failed to detect a file format. Please try to set a format explicitly.",
                    e);
        }
    } else {
        return factory.createCompressorInputStream(format, in);
    }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
@Test
public void testOpenForGeneratedCompression() throws Exception
{
    String[] testFormats = new String[]{
            CompressorStreamFactory.BZIP2,
            CompressorStreamFactory.DEFLATE,
            CompressorStreamFactory.GZIP,
            // CompressorStreamFactory.LZMA, // CompressorException: Compressor: lzma not found.
            // CompressorStreamFactory.PACK200, // Failed to generate compressed file.
            // CompressorStreamFactory.SNAPPY_FRAMED, // CompressorException: Compressor: snappy-framed not found.
            // CompressorStreamFactory.SNAPPY_RAW, // CompressorException: Compressor: snappy-raw not found.
            // CompressorStreamFactory.XZ, // ClassNotFoundException: org.tukaani.xz.FilterOptions
            // CompressorStreamFactory.Z, // CompressorException: Compressor: z not found.
    };

    for (String format : testFormats) {
        TaskSource mockTaskSource = new MockTaskSource(format);
        FileInput mockInput = new MockFileInput(
                getInputStreamAsBuffer(
                        getCompressorInputStream(format, "sample_1.csv")));
        CommonsCompressDecoderPlugin plugin = new CommonsCompressDecoderPlugin();
        FileInput archiveFileInput = plugin.open(mockTaskSource, mockInput);
        verifyContents(archiveFileInput, "1,foo");
    }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
@Test
public void shouldStreamGzipCompressedFile() throws Exception {
    String line;
    final File logFile = new File(ACCESS_LOG_GZIP_FILE);
    final FileInputStream fileInputStream = new FileInputStream(logFile);
    final BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream);
    final CompressorInputStream compressedInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
    final InputStreamReader inputStreamReader = new InputStreamReader(compressedInputStream);
    final BufferedReader reader = new BufferedReader(inputStreamReader);

    while ((line = reader.readLine()) != null) {
        assertTrue(line.contains("localhost"));
    }

    reader.close();
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
@Test
public void shouldStreamBzip2CompressedFile() throws Exception {
    String line;

    final File logFile = new File(ACCESS_LOG_BZIP2_FILE);
    final FileInputStream fileInputStream = new FileInputStream(logFile);
    final BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream);
    final CompressorInputStream compressedInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
    final InputStreamReader inputStreamReader = new InputStreamReader(compressedInputStream);
    final BufferedReader reader = new BufferedReader(inputStreamReader);

    while ((line = reader.readLine()) != null) {
        assertTrue(line.contains("localhost"));
    }

    reader.close();
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
private void testCompressedFile(String compressionType) throws Exception {

    //write data into the stream using the specified compression
    ByteArrayOutputStream bOut = new ByteArrayOutputStream();
    CompressorOutputStream cOut = new CompressorStreamFactory().createCompressorOutputStream(compressionType, bOut);
    cOut.write("StreamSets".getBytes());
    cOut.close();

    //create compression input
    CompressionDataParser.CompressionInputBuilder compressionInputBuilder =
        new CompressionDataParser.CompressionInputBuilder(Compression.COMPRESSED_FILE, null,
            new ByteArrayInputStream(bOut.toByteArray()), "0");
    CompressionDataParser.CompressionInput input = compressionInputBuilder.build();

    //verify
    Assert.assertNotNull(input);
    Assert.assertEquals("myFile::4567", input.wrapOffset("myFile::4567"));
    Assert.assertEquals("myFile::4567", input.wrapRecordId("myFile::4567"));
    InputStream myFile = input.getNextInputStream();
    BufferedReader reader = new BufferedReader(new InputStreamReader(myFile));
    Assert.assertEquals("StreamSets", reader.readLine());
  }
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
/** Unpackages single file content, uses various techniques to uncompress it */
    public static byte[] unCompress(byte[] content) throws IOException {
        //TODO: should probably catch the case if the file is already uncompressed
        try (
                ByteArrayInputStream compressedIn = new ByteArrayInputStream(content);
                CompressorInputStream compressedInStream = new CompressorStreamFactory().createCompressorInputStream(compressedIn);
                ByteArrayOutputStream out = new ByteArrayOutputStream();
                ) {
            final byte[] buffer = new byte[8192];
            int n;
            while ((n = compressedInStream.read(buffer)) != -1) {
                out.write(buffer, 0, n);
            }

            byte[] uncompressed = out.toByteArray();
            return uncompressed;
        } catch (CompressorException e) {
//            System.out.println("General gunzip approach failed trying unzip");
//            e.printStackTrace();
            
            // try unzip approach, given the other ones failed
            return unZip(content);
        }
    }
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
private static Path getArchive(final Path path) throws IOException {
    final Path result;
    // Get the extension
    final String fileName = path.getFileName().toString();
    final String loweredFileName = fileName.toLowerCase(Locale.ENGLISH);
    if (loweredFileName.endsWith(".gz")) {
        String tempFileName = fileName.substring(0, loweredFileName.indexOf(".gz"));
        final int index = tempFileName.lastIndexOf('.');
        if (index > 0) {
            result = Files.createTempFile(tempFileName.substring(0, index), tempFileName.substring(index, tempFileName.length()));
        } else {
            result = Files.createTempFile(tempFileName.substring(0, index), "");
        }
        try (CompressorInputStream in = new CompressorStreamFactory().createCompressorInputStream(new BufferedInputStream(Files.newInputStream(path)))) {
            Files.copy(in, result, StandardCopyOption.REPLACE_EXISTING);
        } catch (CompressorException e) {
            throw new IOException(e);
        }
    } else {
        result = path;
    }
    return result;
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
private void indexInputFile() throws IOException, CompressorException {
  FileInputStream fin = new FileInputStream(input);
  BufferedInputStream in = new BufferedInputStream(fin);
  try (CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(in)) {
    final byte[] buffer = new byte[8192];
    int n;
    while ((n = input.read(buffer)) != -1) {
      String buf = new String(buffer, 0, n);  // TODO: not always correct, we need to wait for line end first?
      String[] lines = buf.split("\n");
      indexLine(lines);
    }
  }
  writeToDisk(1, unigramToCount);
  writeToDisk(2, bigramToCount);
  writeToDisk(3, trigramToCount);
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
private void extract(Language language, String xmlDumpPath) throws IOException, CompressorException {
  try (FileInputStream fis = new FileInputStream(xmlDumpPath);
       BufferedInputStream bis = new BufferedInputStream(fis);
       CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis)) {
    int sentenceCount = 0;
    WikipediaSentenceSource source = new WikipediaSentenceSource(input, language);
    while (source.hasNext()) {
      String sentence = source.next().getText();
      if (skipSentence(sentence)) {
        continue;
      }
      System.out.println(sentence);
      sentenceCount++;
      if (sentenceCount % 1000 == 0) {
        System.err.println("Exporting sentence #" + sentenceCount + "...");
      }
    }
  }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
@Override
protected WritableByteChannel getDirectWritableChannel()
      throws ContentIOException
{
   // Get the raw writer onto the real stream
   OutputStream rawOut = realContentWriter.getContentOutputStream();
   
   try {
      // Wrap that with the requested compression
      CompressorOutputStream compOut = new CompressorStreamFactory()
           .createCompressorOutputStream(compressionType, rawOut);

      logger.info("Compressing " + realContentWriter.getContentUrl() + " with " + compressionType);

      // Turn that into a channel and return
      return Channels.newChannel(compOut);
   }
   catch (CompressorException e)
   {
      throw new AlfrescoRuntimeException("Error compressing", e);
   }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
@Override
public void afterPropertiesSet() throws Exception
{
   // Compression type is optional, use GZip if in doubt
   if (compressionType == null || compressionType.isEmpty())
   {
      compressionType = CompressorStreamFactory.GZIP;
   }
   
   // Real Content Store and MimeTypes must be given
   if (compressMimeTypes == null || compressMimeTypes.isEmpty())
   {
      throw new IllegalArgumentException("'compressMimeTypes' must be given");
   }
   if (realContentStore == null)
   {
      throw new IllegalArgumentException("'realContentStore' must be given");
   }
   if (mimetypeService == null)
   {
      throw new IllegalArgumentException("'mimetypeService' must be given");
   }
}
 

import org.apache.commons.compress.compressors.CompressorStreamFactory; //導入依賴的package包/類
static Content buildPackages(final Collection<Map<String, String>> entries) throws IOException {
  CompressorStreamFactory compressorStreamFactory = new CompressorStreamFactory();
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  try (CompressorOutputStream cos = compressorStreamFactory.createCompressorOutputStream(GZIP, os)) {
    try (OutputStreamWriter writer = new OutputStreamWriter(cos, UTF_8)) {
      for (Map<String, String> entry : entries) {
        InternetHeaders headers = new InternetHeaders();
        headers.addHeader(P_PACKAGE, entry.get(P_PACKAGE));
        headers.addHeader(P_VERSION, entry.get(P_VERSION));
        headers.addHeader(P_DEPENDS, entry.get(P_DEPENDS));
        headers.addHeader(P_IMPORTS, entry.get(P_IMPORTS));
        headers.addHeader(P_SUGGESTS, entry.get(P_SUGGESTS));
        headers.addHeader(P_LICENSE, entry.get(P_LICENSE));
        headers.addHeader(P_NEEDS_COMPILATION, entry.get(P_NEEDS_COMPILATION));
        Enumeration<String> headerLines = headers.getAllHeaderLines();
        while (headerLines.hasMoreElements()) {
          String line = headerLines.nextElement();
          writer.write(line, 0, line.length());
          writer.write('\n');
        }
        writer.write('\n');
      }
    }
  }
  catch ( CompressorException e ) {
    throw new RException(null, e);
  }
  return new Content(new BytesPayload(os.toByteArray(), "application/x-gzip"));
}