Java OfflineSorter类代码示例

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Override
public BytesRefIterator iterator() throws IOException {
  if (sorted == null) {
    closeWriter();
    
    sorted = File.createTempFile("RefSorter-", ".sorted",
        OfflineSorter.defaultTempDir());
    sort.sort(input, sorted);
    
    input.delete();
    input = null;
  }
  
  return new ByteSequenceIterator(new OfflineSorter.ByteSequencesReader(sorted),
      sort.getComparator());
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
/**
 * Will buffer all sequences to a temporary file and then sort (all on-disk).
 */
public ExternalRefSorter(OfflineSorter sort) throws IOException {
  this.sort = sort;
  this.input = File.createTempFile("RefSorter-", ".raw",
      OfflineSorter.defaultTempDir());
  this.writer = new OfflineSorter.ByteSequencesWriter(input);
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private ByteSequencesReader sort() throws IOException {
  String prefix = getClass().getSimpleName();
  File directory = OfflineSorter.defaultTempDir();
  tempInput = File.createTempFile(prefix, ".input", directory);
  tempSorted = File.createTempFile(prefix, ".sorted", directory);
  
  final ByteSequencesWriter writer = new ByteSequencesWriter(tempInput);
  boolean success = false;
  try {
    BytesRef spare;
    byte[] buffer = new byte[0];
    ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);

    while ((spare = source.next()) != null) {
      encode(writer, output, buffer, spare, source.weight());
    }
    writer.close();
    new OfflineSorter(tieBreakByCostComparator).sort(tempInput, tempSorted);
    ByteSequencesReader reader = new ByteSequencesReader(tempSorted);
    success = true;
    return reader;
    
  } finally {
    if (success) {
      IOUtils.close(writer);
    } else {
      try {
        IOUtils.closeWhileHandlingException(writer);
      } finally {
        close();
      }
    }
  }
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private ByteSequencesReader sort() throws IOException {
  String prefix = getClass().getSimpleName();
  File directory = OfflineSorter.defaultTempDir();
  tempInput = File.createTempFile(prefix, ".input", directory);
  tempSorted = File.createTempFile(prefix, ".sorted", directory);
  
  final OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
  boolean success = false;
  try {
    BytesRef spare;
    byte[] buffer = new byte[0];
    ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);

    while ((spare = source.next()) != null) {
      encode(writer, output, buffer, spare, source.payload(), source.contexts(), source.weight());
    }
    writer.close();
    new OfflineSorter(tieBreakByCostComparator).sort(tempInput, tempSorted);
    ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(tempSorted);
    success = true;
    return reader;
    
  } finally {
    if (success) {
      IOUtils.close(writer);
    } else {
      try {
        IOUtils.closeWhileHandlingException(writer);
      } finally {
        close();
      }
    }
  }
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
  File input = new File("/home/dweiss/tmp/shuffled.dict");

  int buckets = 20;
  int shareMaxTail = 10;

  ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
  FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);

  BufferedReader reader = new BufferedReader(
      new InputStreamReader(
          new FileInputStream(input), StandardCharsets.UTF_8));
  
  BytesRefBuilder scratch = new BytesRefBuilder();
  String line;
  int count = 0;
  while ((line = reader.readLine()) != null) {
    scratch.copyChars(line);
    builder.add(scratch.get(), count % buckets);
    if ((count++ % 100000) == 0) {
      System.err.println("Line: " + count);
    }
  }

  System.out.println("Building FSTCompletion.");
  FSTCompletion completion = builder.build();

  File fstFile = new File("completion.fst");
  System.out.println("Done. Writing automaton: " + fstFile.getAbsolutePath());
  completion.getFST().save(fstFile);
  sorter.close();
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
/**
 * Check sorting data on an instance of {@link OfflineSorter}.
 */
private SortInfo checkSort(OfflineSorter sort, byte[][] data) throws IOException {
  File unsorted = writeAll("unsorted", data);

  Arrays.sort(data, unsignedByteOrderComparator);
  File golden = writeAll("golden", data);

  File sorted = new File(tempDir, "sorted");
  SortInfo sortInfo = sort.sort(unsorted, sorted);
  //System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
  //System.out.println(sortInfo);

  assertFilesIdentical(golden, sorted);
  return sortInfo;
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private File writeAll(String name, byte[][] data) throws IOException {
  File file = new File(tempDir, name);
  ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(file);
  for (byte [] datum : data) {
    w.write(datum);
  }
  w.close();
  return file;
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Test
public void testExternalRefSorter() throws Exception {
  ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
  check(s);
  s.close();
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testEmpty() throws Exception {
  checkSort(new OfflineSorter(), new byte [][] {});
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testSingleLine() throws Exception {
  checkSort(new OfflineSorter(), new byte [][] {
      "Single line only.".getBytes(StandardCharsets.UTF_8)
  });
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testIntermediateMerges() throws Exception {
  // Sort 20 mb worth of data with 1mb buffer, binary merging.
  SortInfo info = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), 2), 
      generateRandom((int)OfflineSorter.MB * 20));
  assertTrue(info.mergeRounds > 10);
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testSmallRandom() throws Exception {
  // Sort 20 mb worth of data with 1mb buffer.
  SortInfo sortInfo = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES), 
      generateRandom((int)OfflineSorter.MB * 20));
  assertEquals(1, sortInfo.mergeRounds);
}
 

import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Nightly
public void testLargerRandom() throws Exception {
  // Sort 100MB worth of data with 15mb buffer.
  checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES), 
      generateRandom((int)OfflineSorter.MB * 100));
}