本文整理汇总了Java中org.apache.lucene.util.OfflineSorter类的典型用法代码示例。如果您正苦于以下问题:Java OfflineSorter类的具体用法?Java OfflineSorter怎么用?Java OfflineSorter使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
OfflineSorter类属于org.apache.lucene.util包,在下文中一共展示了OfflineSorter类的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: iterator
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Override
public BytesRefIterator iterator() throws IOException {
if (sorted == null) {
closeWriter();
sorted = File.createTempFile("RefSorter-", ".sorted",
OfflineSorter.defaultTempDir());
sort.sort(input, sorted);
input.delete();
input = null;
}
return new ByteSequenceIterator(new OfflineSorter.ByteSequencesReader(sorted),
sort.getComparator());
}
示例2: ExternalRefSorter
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
/**
* Will buffer all sequences to a temporary file and then sort (all on-disk).
*/
public ExternalRefSorter(OfflineSorter sort) throws IOException {
this.sort = sort;
this.input = File.createTempFile("RefSorter-", ".raw",
OfflineSorter.defaultTempDir());
this.writer = new OfflineSorter.ByteSequencesWriter(input);
}
示例3: sort
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private ByteSequencesReader sort() throws IOException {
String prefix = getClass().getSimpleName();
File directory = OfflineSorter.defaultTempDir();
tempInput = File.createTempFile(prefix, ".input", directory);
tempSorted = File.createTempFile(prefix, ".sorted", directory);
final ByteSequencesWriter writer = new ByteSequencesWriter(tempInput);
boolean success = false;
try {
BytesRef spare;
byte[] buffer = new byte[0];
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
while ((spare = source.next()) != null) {
encode(writer, output, buffer, spare, source.weight());
}
writer.close();
new OfflineSorter(tieBreakByCostComparator).sort(tempInput, tempSorted);
ByteSequencesReader reader = new ByteSequencesReader(tempSorted);
success = true;
return reader;
} finally {
if (success) {
IOUtils.close(writer);
} else {
try {
IOUtils.closeWhileHandlingException(writer);
} finally {
close();
}
}
}
}
示例4: sort
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private ByteSequencesReader sort() throws IOException {
String prefix = getClass().getSimpleName();
File directory = OfflineSorter.defaultTempDir();
tempInput = File.createTempFile(prefix, ".input", directory);
tempSorted = File.createTempFile(prefix, ".sorted", directory);
final OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
boolean success = false;
try {
BytesRef spare;
byte[] buffer = new byte[0];
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
while ((spare = source.next()) != null) {
encode(writer, output, buffer, spare, source.payload(), source.contexts(), source.weight());
}
writer.close();
new OfflineSorter(tieBreakByCostComparator).sort(tempInput, tempSorted);
ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(tempSorted);
success = true;
return reader;
} finally {
if (success) {
IOUtils.close(writer);
} else {
try {
IOUtils.closeWhileHandlingException(writer);
} finally {
close();
}
}
}
}
示例5: main
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public static void main(String[] args) throws IOException {
File input = new File("/home/dweiss/tmp/shuffled.dict");
int buckets = 20;
int shareMaxTail = 10;
ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);
BufferedReader reader = new BufferedReader(
new InputStreamReader(
new FileInputStream(input), StandardCharsets.UTF_8));
BytesRefBuilder scratch = new BytesRefBuilder();
String line;
int count = 0;
while ((line = reader.readLine()) != null) {
scratch.copyChars(line);
builder.add(scratch.get(), count % buckets);
if ((count++ % 100000) == 0) {
System.err.println("Line: " + count);
}
}
System.out.println("Building FSTCompletion.");
FSTCompletion completion = builder.build();
File fstFile = new File("completion.fst");
System.out.println("Done. Writing automaton: " + fstFile.getAbsolutePath());
completion.getFST().save(fstFile);
sorter.close();
}
示例6: checkSort
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
/**
* Check sorting data on an instance of {@link OfflineSorter}.
*/
private SortInfo checkSort(OfflineSorter sort, byte[][] data) throws IOException {
File unsorted = writeAll("unsorted", data);
Arrays.sort(data, unsignedByteOrderComparator);
File golden = writeAll("golden", data);
File sorted = new File(tempDir, "sorted");
SortInfo sortInfo = sort.sort(unsorted, sorted);
//System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
//System.out.println(sortInfo);
assertFilesIdentical(golden, sorted);
return sortInfo;
}
示例7: writeAll
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
private File writeAll(String name, byte[][] data) throws IOException {
File file = new File(tempDir, name);
ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(file);
for (byte [] datum : data) {
w.write(datum);
}
w.close();
return file;
}
示例8: testExternalRefSorter
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Test
public void testExternalRefSorter() throws Exception {
ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
check(s);
s.close();
}
示例9: testEmpty
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testEmpty() throws Exception {
checkSort(new OfflineSorter(), new byte [][] {});
}
示例10: testSingleLine
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testSingleLine() throws Exception {
checkSort(new OfflineSorter(), new byte [][] {
"Single line only.".getBytes(StandardCharsets.UTF_8)
});
}
示例11: testIntermediateMerges
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
SortInfo info = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), 2),
generateRandom((int)OfflineSorter.MB * 20));
assertTrue(info.mergeRounds > 10);
}
示例12: testSmallRandom
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
SortInfo sortInfo = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES),
generateRandom((int)OfflineSorter.MB * 20));
assertEquals(1, sortInfo.mergeRounds);
}
示例13: testLargerRandom
import org.apache.lucene.util.OfflineSorter; //导入依赖的package包/类
@Nightly
public void testLargerRandom() throws Exception {
// Sort 100MB worth of data with 15mb buffer.
checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES),
generateRandom((int)OfflineSorter.MB * 100));
}