Java InputIterator类代码示例

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
/**
 * Rebuild a suggestion index from the document index.
 *
 * This method iterates through the entire document index and makes sure that only unique titles
 * are indexed.
 *
 * @param indexRoot The parent directory inside which both the document index and the suggestion
 *                  index lives.
 * @throws IOException
 */
public static void rebuild(String indexRoot) throws IOException {
  Path indexRootPath = Paths.get(indexRoot);
  Path suggestionPath = getSuggestionIndexPath(indexRootPath);

  // Delete the suggestion index if it exists.
  if (Files.exists(suggestionPath)) {
    Util.deletePath(suggestionPath);
  }

  // Create the suggestion index.
  Analyzer analyzer = Indexer.getAnalyzer();
  Directory suggestionDir = FSDirectory.open(getSuggestionIndexPath(indexRootPath));
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(suggestionDir, analyzer);

  // Open the document index.
  Directory indexDir = FSDirectory.open(Indexer.getMainIndexPath(indexRootPath));
  IndexReader reader = DirectoryReader.open(indexDir);

  // Get a document iterator.
  DocumentDictionary docDict = new DocumentDictionary(reader, Indexer.TITLE_FIELD_NAME, null);
  InputIterator iterator = docDict.getEntryIterator();
  Set<BytesRef> titleSet = new HashSet<>();
  BytesRef next;
  while ((next = iterator.next()) != null) {
    if (titleSet.contains(next)) {
      continue;
    }

    titleSet.add(next);
    suggester.add(next, null, 0, null);
  }

  reader.close();

  suggester.commit();
  suggester.close();
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public void build(InputIterator iterator) throws IOException {
  if (iterator.hasPayloads()) {
    throw new IllegalArgumentException("this suggester doesn't support payloads");
  }
  BytesRef scratch = new BytesRef();
  InputIterator iter = new WFSTInputIterator(iterator);
  IntsRef scratchInts = new IntsRef();
  BytesRef previous = null;
  PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
  Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
  while ((scratch = iter.next()) != null) {
    long cost = iter.weight();
    
    if (previous == null) {
      previous = new BytesRef();
    } else if (scratch.equals(previous)) {
      continue; // for duplicate suggestions, the best weight is actually
                // added
    }
    Util.toIntsRef(scratch, scratchInts);
    builder.add(scratchInts, cost);
    previous.copyBytes(scratch);
  }
  fst = builder.finish();
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public final InputIterator getEntryIterator() throws IOException {
  final Terms terms = MultiFields.getTerms(reader, field);
  if (terms != null) {
    return new InputIterator.InputIteratorWrapper(terms.iterator(null));
  } else {
    return InputIterator.EMPTY;
  }
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
private static Lookup setupSuggester_Analyzing() {

		setupIndexReader();

		final Lookup suggester[] = new AnalyzingSuggester[1];

		Display.getDefault().syncExec(new Runnable() {
			@Override
			public void run() {

				BusyIndicator.showWhile(Display.getDefault(), new Runnable() {
					@Override
					public void run() {

						try {

							final Analyzer queryAnalyzer = new StandardAnalyzer(new CharArraySet(0, true));
							final InputIterator termIterator = createTermIterator();

							suggester[0] = new AnalyzingSuggester(queryAnalyzer);
							suggester[0].build(termIterator);

						} catch (final Exception e) {
							StatusUtil.showStatus(e);
						}
					}
				});
			}
		});

		return suggester[0];
	}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
/**
 * Like build(), but without flushing the old entries, and *ignores duplicate entries*
 * 
 * @param dict
 * @throws IOException
 */
public void add(Dictionary dict) throws IOException {
  InputIterator iter = dict.getEntryIterator();
  BytesRef text;
  while ((text = iter.next()) != null) {
    if (lookup(text.utf8ToString(), 1, true, false).size() > 0) {
      continue;
    }
    add(text, iter.contexts(), iter.weight(), iter.payload());
  }
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public void build(InputIterator iterator) throws IOException {
  if (iterator.hasPayloads()) {
    throw new IllegalArgumentException("this suggester doesn't support payloads");
  }
  count = 0;
  BytesRef scratch = new BytesRef();
  InputIterator iter = new WFSTInputIterator(iterator);
  IntsRef scratchInts = new IntsRef();
  BytesRef previous = null;
  PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
  Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
  while ((scratch = iter.next()) != null) {
    long cost = iter.weight();
    
    if (previous == null) {
      previous = new BytesRef();
    } else if (scratch.equals(previous)) {
      continue; // for duplicate suggestions, the best weight is actually
                // added
    }
    Util.toIntsRef(scratch, scratchInts);
    builder.add(scratchInts, cost);
    previous.copyBytes(scratch);
    count++;
  }
  fst = builder.finish();
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public void build(InputIterator iterator) throws IOException {
  build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
WFSTInputIterator(InputIterator source) throws IOException {
  super(source);
  assert source.hasPayloads() == false;
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public InputIterator getEntryIterator() throws IOException {
  return new InputIterator.InputIteratorWrapper(new FileIterator());
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public final InputIterator getEntryIterator() throws IOException {
  return new HighFrequencyIterator();
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Ignore
public void testWiki() throws Exception {
  final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt", false);
  // Skip header:
  lfd.nextDoc();
  FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
  sug.build(new InputIterator() {

      private int count;

      @Override
      public long weight() {
        return 1;
      }

      @Override
      public Comparator<BytesRef> getComparator() {
        return null;
      }

      @Override
      public BytesRef next() {
        Document doc;
        try {
          doc = lfd.nextDoc();
        } catch (IOException ioe) {
          throw new RuntimeException(ioe);
        }
        if (doc == null) {
          return null;
        }
        if (count++ == 10000) {
          return null;
        }
        return new BytesRef(doc.get("body"));
      }

      @Override
      public BytesRef payload() {
        return null;
      }

      @Override
      public boolean hasPayloads() {
        return false;
      }

      @Override
      public Set<BytesRef> contexts() {
        return null;
      }

      @Override
      public boolean hasContexts() {
        return false;
      }
    });
  if (VERBOSE) {
    System.out.println(sug.ramBytesUsed() + " bytes");

    List<LookupResult> results = sug.lookup("general r", 10);
    System.out.println("results:");
    for(LookupResult result : results) {
      System.out.println("  " + result);
    }
  }
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public InputIterator getEntryIterator() throws IOException {
    return new InputIterator.InputIteratorWrapper(new FileIterator());
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
private static InputIterator createTermIterator() throws IOException {

		final TermFreqIteratorListWrapper inputIterator = new TermFreqIteratorListWrapper();

		final List<AtomicReaderContext> leaves = _indexReader.leaves();

		for (final AtomicReaderContext readerContext : leaves) {

			final AtomicReader reader = readerContext.reader();
			final Fields fields = reader.fields();

			for (final String field : fields) {

				if (field.equals(SEARCH_FIELD_DESCRIPTION) || field.equals(SEARCH_FIELD_TITLE)) {

					final Terms terms = fields.terms(field);
					final TermsEnum termsEnum = terms.iterator(null);

					inputIterator.add(termsEnum);
				}
			}
		}

		return inputIterator;
	}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Override
public InputIterator getEntryIterator() throws IOException {
  return new MultiInputIterator();
}
 

import org.apache.lucene.search.suggest.InputIterator; //导入依赖的package包/类
@Ignore
public void testWiki() throws Exception {
  final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt", false);
  // Skip header:
  lfd.nextDoc();
  FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
  sug.build(new InputIterator() {

      private int count;

      @Override
      public long weight() {
        return 1;
      }

      @Override
      public Comparator<BytesRef> getComparator() {
        return null;
      }

      @Override
      public BytesRef next() {
        Document doc;
        try {
          doc = lfd.nextDoc();
        } catch (IOException ioe) {
          throw new RuntimeException(ioe);
        }
        if (doc == null) {
          return null;
        }
        if (count++ == 10000) {
          return null;
        }
        return new BytesRef(doc.get("body"));
      }

      @Override
      public BytesRef payload() {
        return null;
      }

      @Override
      public boolean hasPayloads() {
        return false;
      }
    });
  if (VERBOSE) {
    System.out.println(sug.sizeInBytes() + " bytes");

    List<LookupResult> results = sug.lookup("general r", 10);
    System.out.println("results:");
    for(LookupResult result : results) {
      System.out.println("  " + result);
    }
  }
}