Java StreamUtils类代码示例

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
public WriteLineDocTask(PerfRunData runData) throws Exception {
  super(runData);
  Config config = runData.getConfig();
  fname = config.get("line.file.out", null);
  if (fname == null) {
    throw new IllegalArgumentException("line.file.out must be set");
  }
  OutputStream out = StreamUtils.outputStream(new File(fname));
  lineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8), StreamUtils.BUFFER_SIZE));
  docMaker = runData.getDocMaker();
  
  // init fields 
  String f2r = config.get("line.fields",null);
  if (f2r == null) {
    fieldsToWrite = DEFAULT_FIELDS;
  } else {
    if (f2r.indexOf(SEP)>=0) {
      throw new IllegalArgumentException("line.fields "+f2r+" should not contain the separator char: "+SEP);
    }
    fieldsToWrite = f2r.split(","); 
  }
  
  // init sufficient fields
  sufficientFields = new boolean[fieldsToWrite.length];
  String suff = config.get("sufficient.fields",DEFAULT_SUFFICIENT_FIELDS);
  if (",".equals(suff)) {
    checkSufficientFields = false;
  } else {
    checkSufficientFields = true;
    HashSet<String> sf = new HashSet<>(Arrays.asList(suff.split(",")));
    for (int i=0; i<fieldsToWrite.length; i++) {
      if (sf.contains(fieldsToWrite[i])) {
        sufficientFields[i] = true;
      }
    }
  }
  
  writeHeader(lineFileOut);
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
private synchronized void openFile() {
  try {
    if (reader != null) {
      reader.close();
    }
    InputStream is = StreamUtils.inputStream(file);
    reader = new BufferedReader(new InputStreamReader(is, encoding), StreamUtils.BUFFER_SIZE);
    if (skipHeaderLine) {
      reader.readLine(); // skip one line - the header line - already handled that info
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
void openNextFile() throws NoMoreDataException, IOException {
  close();
  currPathType = null;
  while (true) {
    if (nextFile >= inputFiles.size()) { 
      // exhausted files, start a new round, unless forever set to false.
      if (!forever) {
        throw new NoMoreDataException();
      }
      nextFile = 0;
      iteration++;
    }
    File f = inputFiles.get(nextFile++);
    if (verbose) {
      System.out.println("opening: " + f + " length: " + f.length());
    }
    try {
      InputStream inputStream = StreamUtils.inputStream(f); // support either gzip, bzip2, or regular text file, by extension  
      reader = new BufferedReader(new InputStreamReader(inputStream, encoding), StreamUtils.BUFFER_SIZE);
      currPathType = TrecDocParser.pathType(f);
      return;
    } catch (Exception e) {
      if (verbose) {
        System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
        continue;
      }
      throw new NoMoreDataException();
    }
  }
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
public WriteLineDocTask(PerfRunData runData) throws Exception {
  super(runData);
  Config config = runData.getConfig();
  fname = config.get("line.file.out", null);
  if (fname == null) {
    throw new IllegalArgumentException("line.file.out must be set");
  }
  OutputStream out = StreamUtils.outputStream(new File(fname));
  lineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, "UTF-8"), StreamUtils.BUFFER_SIZE));
  docMaker = runData.getDocMaker();
  
  // init fields 
  String f2r = config.get("line.fields",null);
  if (f2r == null) {
    fieldsToWrite = DEFAULT_FIELDS;
  } else {
    if (f2r.indexOf(SEP)>=0) {
      throw new IllegalArgumentException("line.fields "+f2r+" should not contain the separator char: "+SEP);
    }
    fieldsToWrite = f2r.split(","); 
  }
  
  // init sufficient fields
  sufficientFields = new boolean[fieldsToWrite.length];
  String suff = config.get("sufficient.fields",DEFAULT_SUFFICIENT_FIELDS);
  if (",".equals(suff)) {
    checkSufficientFields = false;
  } else {
    checkSufficientFields = true;
    HashSet<String> sf = new HashSet<String>(Arrays.asList(suff.split(",")));
    for (int i=0; i<fieldsToWrite.length; i++) {
      if (sf.contains(fieldsToWrite[i])) {
        sufficientFields[i] = true;
      }
    }
  }
  
  writeHeader(lineFileOut);
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
private void assertReadText(File f) throws Exception {
  InputStream ir = StreamUtils.inputStream(f);
  InputStreamReader in = new InputStreamReader(ir, IOUtils.CHARSET_UTF_8);
  BufferedReader r = new BufferedReader(in);
  String line = r.readLine();
  assertEquals("Wrong text found in "+f.getName(), TEXT, line);
  r.close();
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
void openNextFile() throws NoMoreDataException, IOException {
  close();

  while (true) {
    if (nextFile >= inputFiles.size()) { 
      // exhausted files, start a new round, unless forever set to false.
      if (!forever) {
        throw new NoMoreDataException();
      }
      nextFile = 0;
      iteration++;
    }
    File f = inputFiles.get(nextFile++);
    if (verbose) {
      System.out.println("opening: " + f + " length: " + f.length());
    }
    try {
      // supports gzip, bzip2, or regular text file, extension is used to detect
      InputStream inputStream = StreamUtils.inputStream(f);   
      reader = new DataInputStream(inputStream);
      return;
    } catch (Exception e) {
      if (verbose) {
        System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
        continue;
      }
      throw new NoMoreDataException();
    }
  }
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
public WriteEnwikiLineDocTask(PerfRunData runData) throws Exception {
  super(runData);
  OutputStream out = StreamUtils.outputStream(categoriesLineFile(new File(fname)));
  categoryLineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8), StreamUtils.BUFFER_SIZE));
  writeHeader(categoryLineFileOut);
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
/** Open the input stream. */
protected InputStream openInputStream() throws IOException {
  return StreamUtils.inputStream(file);
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
public WriteEnwikiLineDocTask(PerfRunData runData) throws Exception {
  super(runData);
  OutputStream out = StreamUtils.outputStream(categoriesLineFile(new File(fname)));
  categoryLineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, "UTF-8"), StreamUtils.BUFFER_SIZE));
  writeHeader(categoryLineFileOut);
}
 

import org.apache.lucene.benchmark.byTask.utils.StreamUtils; //导入依赖的package包/类
private File autoOutFile(String ext) throws Exception {
  File f = new File(testDir,"testfile." +  ext);
  OutputStream os = StreamUtils.outputStream(f);
  writeText(os);
  return f;
}