Java TokenSources.getAnyTokenStream方法代码示例

import org.apache.lucene.search.highlight.TokenSources; //导入方法依赖的package包/类
public void testHits() throws Exception {
    IndexSearcher searcher = new IndexSearcher(TestUtil.getBookIndexDirectory());
    TermQuery query = new TermQuery(new Term("title", "action"));
    TopDocs hits = searcher.search(query, 10);

    QueryScorer scorer = new QueryScorer(query, "title");
    Highlighter highlighter = new Highlighter(scorer);
    highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));

    Analyzer analyzer = new SimpleAnalyzer();

    for (ScoreDoc sd : hits.scoreDocs) {
        StoredDocument doc = searcher.doc(sd.doc);
        String title = doc.get("title");

        TokenStream stream = TokenSources.getAnyTokenStream(searcher.getIndexReader(), sd.doc, "title", doc,
                analyzer);
        String fragment = highlighter.getBestFragment(stream, title);

        LOGGER.info(fragment);
    }
}
 

import org.apache.lucene.search.highlight.TokenSources; //导入方法依赖的package包/类
@Override
protected BenchmarkHighlighter getBenchmarkHighlighter(Query q){
  highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
  highlighter.setMaxDocCharsToAnalyze(maxDocCharsToAnalyze);
  return new BenchmarkHighlighter(){
    @Override
    public int doHighlight(IndexReader reader, int doc, String field,
        Document document, Analyzer analyzer, String text) throws Exception {
      TokenStream ts = TokenSources.getAnyTokenStream(reader, doc, field, document, analyzer);
      TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFrags);
      return frag != null ? frag.length : 0;
    }
  };
}
 

import org.apache.lucene.search.highlight.TokenSources; //导入方法依赖的package包/类
@Override
public BenchmarkHighlighter getBenchmarkHighlighter(Query q) {
  highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
  return new BenchmarkHighlighter() {
    @Override
    public int doHighlight(IndexReader reader, int doc, String field, Document document, Analyzer analyzer, String text) throws Exception {
      TokenStream ts = TokenSources.getAnyTokenStream(reader, doc, field, document, analyzer);
      TextFragment[] frag = highlighter.getBestTextFragments(ts, text, mergeContiguous, maxFrags);
      numHighlightedResults += frag != null ? frag.length : 0;
      return frag != null ? frag.length : 0;
    }
  };
}
 

import org.apache.lucene.search.highlight.TokenSources; //导入方法依赖的package包/类
private String searchData(String key) throws IOException, ParseException, InvalidTokenOffsetsException {
	Directory directory = FSDirectory.open(new File(filePath));
	IndexSearcher indexSearcher = new IndexSearcher(directory);
	
	QueryParser queryParser = new QueryParser(Version.LUCENE_31, "foods",
			new SmartChineseAnalyzer(Version.LUCENE_31, true));
	//queryParser.setDefaultOperator(Operator.AND);

	Query query = queryParser.parse(key);
	TopDocs docs = indexSearcher.search(query, 10);
	
	QueryScorer queryScorer = new QueryScorer(query, "foods");
	Highlighter highlighter = new Highlighter(queryScorer);
	highlighter.setTextFragmenter(new SimpleSpanFragmenter(queryScorer));


	List<SearchResult> searchResults = new ArrayList<SearchResult>();
	
	if (docs != null) {
		for (ScoreDoc scoreDoc : docs.scoreDocs) {
			Document doc = indexSearcher.doc(scoreDoc.doc);
			TokenStream tokenStream = TokenSources.getAnyTokenStream(
					indexSearcher.getIndexReader(), scoreDoc.doc, "foods", doc, 
					new SmartChineseAnalyzer(Version.LUCENE_31, true));
			SearchResult searchResult = new SearchResult();
			searchResult.setRestaurantId(Long.valueOf(doc.get("id")));
			searchResult.setRestaurantName(doc.get("restaurant_name"));
			searchResult.setKey(key);
			searchResult.setFoods(Arrays.asList(highlighter.
					getBestFragment(tokenStream, doc.get("foods")).split(" ")));
			searchResults.add(searchResult);
		}
	} else {
		searchResults = null;
	}
	
	indexSearcher.close();
	directory.close();
	
	return new Gson().toJson(searchResults);
}
 

import org.apache.lucene.search.highlight.TokenSources; //导入方法依赖的package包/类
/**
 * NOTE: This method will not preserve the correct field types.
 * 
 * @param preTag
 * @param postTag
 */
public static Document highlight(int docId, Document document, Query query, FieldManager fieldManager,
    IndexReader reader, String preTag, String postTag) throws IOException, InvalidTokenOffsetsException {

  String fieldLessFieldName = fieldManager.getFieldLessFieldName();

  Query fixedQuery = fixSuperQuery(query, null, fieldLessFieldName);

  Analyzer analyzer = fieldManager.getAnalyzerForQuery();

  SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(preTag, postTag);
  Document result = new Document();
  for (IndexableField f : document) {
    String name = f.name();
    if (fieldLessFieldName.equals(name) || FIELDS_NOT_TO_HIGHLIGHT.contains(name)) {
      result.add(f);
      continue;
    }
    String text = f.stringValue();
    Number numericValue = f.numericValue();

    Query fieldFixedQuery;
    if (fieldManager.isFieldLessIndexed(name)) {
      fieldFixedQuery = fixSuperQuery(query, name, fieldLessFieldName);
    } else {
      fieldFixedQuery = fixedQuery;
    }

    if (numericValue != null) {
      if (shouldNumberBeHighlighted(name, numericValue, fieldFixedQuery)) {
        String numberHighlight = preTag + text + postTag;
        result.add(new StringField(name, numberHighlight, Store.YES));
      }
    } else {
      Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(fieldFixedQuery, name));
      TokenStream tokenStream = TokenSources.getAnyTokenStream(reader, docId, name, analyzer);
      TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, false, 10);
      for (int j = 0; j < frag.length; j++) {
        if ((frag[j] != null) && (frag[j].getScore() > 0)) {
          result.add(new StringField(name, frag[j].toString(), Store.YES));
        }
      }
    }
  }
  return result;
}