Java CommonTermsQuery.add方法代码示例

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testCommonTermsQuery() throws IOException {
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    Analyzer analyzer = new StandardAnalyzer();
    IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
    CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fox"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testExtractQueryMetadata_commonTermsQuery() {
    CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
    commonTermsQuery.add(new Term("_field", "_term1"));
    commonTermsQuery.add(new Term("_field", "_term2"));
    Result result = analyze(commonTermsQuery);
    assertThat(result.verified, is(false));
    List<Term> terms = new ArrayList<>(result.terms);
    Collections.sort(terms);
    assertThat(terms.size(), equalTo(2));
    assertThat(terms.get(0).field(), equalTo("_field"));
    assertThat(terms.get(0).text(), equalTo("_term1"));
    assertThat(terms.get(1).field(), equalTo("_field"));
    assertThat(terms.get(1).text(), equalTo("_term2"));
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testHighlightingCommonTermsQuery() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
  query.add(new Term(FIELD_NAME, "this"));
  query.add(new Term(FIELD_NAME, "long"));
  query.add(new Term(FIELD_NAME, "very"));

  searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
  Highlighter highlighter = new Highlighter(scorer);

  Document doc = searcher.doc(hits.scoreDocs[0].doc);
  String storedField = doc.get(FIELD_NAME);

  TokenStream stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[0].doc, FIELD_NAME, doc, analyzer);
  Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
  highlighter.setTextFragmenter(fragmenter);
  String fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
  
  doc = searcher.doc(hits.scoreDocs[1].doc);
  storedField = doc.get(FIELD_NAME);

  stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
  highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
  fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testHighlightingCommonTermsQuery() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);
  query.add(new Term(FIELD_NAME, "this"));
  query.add(new Term(FIELD_NAME, "long"));
  query.add(new Term(FIELD_NAME, "very"));

  searcher = new IndexSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
  Highlighter highlighter = new Highlighter(scorer);

  Document doc = searcher.doc(hits.scoreDocs[0].doc);
  String storedField = doc.get(FIELD_NAME);

  TokenStream stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[0].doc, FIELD_NAME, doc, analyzer);
  Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
  highlighter.setTextFragmenter(fragmenter);
  String fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("Hello <B>this</B> is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);
  
  doc = searcher.doc(hits.scoreDocs[1].doc);
  storedField = doc.get(FIELD_NAME);

  stream = TokenSources.getAnyTokenStream(searcher
      .getIndexReader(), hits.scoreDocs[1].doc, FIELD_NAME, doc, analyzer);
  highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));
  fragment = highlighter.getBestFragment(stream, storedField);
  assertEquals("<B>This</B> piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public LuceneResponse similarDocuments(String identifier) throws Throwable {
    SearcherAndTaxonomy reference = data.getManager().acquire();
    try {
        Query idQuery = new TermQuery(new Term(ID_FIELD, identifier));
        TopDocs topDocs = reference.searcher.search(idQuery, 1);
        if (topDocs.totalHits == 0)
            return new LuceneResponse(0);
        int docId = topDocs.scoreDocs[0].doc;
        IndexReader reader = reference.searcher.getIndexReader();
        CommonTermsQuery commonQuery = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD, 0.1f);
        Fields termVectors = reader.getTermVectors(docId);
        if (termVectors == null)
            return new LuceneResponse(0);
        for (String field : termVectors) {
            TermsEnum iterator = termVectors.terms(field).iterator();
            BytesRef b;
            while ((b = iterator.next()) != null) {
                Term term = new Term(field, b.utf8ToString());
                commonQuery.add(term);
            }
        }
        BooleanQuery.Builder query = new BooleanQuery.Builder();
        query.add(idQuery, Occur.MUST_NOT);
        query.add(commonQuery, Occur.MUST);
        return executeQuery(query.build());
    } finally {
        data.getManager().release(reference);
    }
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testDuelSpecificQueries() throws Exception {
    List<ParseContext.Document> documents = new ArrayList<>();

    CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    commonTermsQuery.add(new Term("field", "quick"));
    commonTermsQuery.add(new Term("field", "brown"));
    commonTermsQuery.add(new Term("field", "fox"));
    addQuery(commonTermsQuery, documents);

    BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
            new Term("field", "brown"), new Term("field", "fox")}, false);
    addQuery(blendedTermQuery, documents);

    SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true)
            .addClause(new SpanTermQuery(new Term("field", "quick")))
            .addClause(new SpanTermQuery(new Term("field", "brown")))
            .addClause(new SpanTermQuery(new Term("field", "fox")))
            .build();
    addQuery(spanNearQuery, documents);

    SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true)
            .addClause(new SpanTermQuery(new Term("field", "the")))
            .addClause(new SpanTermQuery(new Term("field", "lazy")))
            .addClause(new SpanTermQuery(new Term("field", "doc")))
            .build();
    SpanOrQuery spanOrQuery = new SpanOrQuery(
            spanNearQuery,
            spanNearQuery2
    );
    addQuery(spanOrQuery, documents);

    SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery);
    addQuery(spanNotQuery, documents);

    long lowerLong = randomIntBetween(0, 256);
    long upperLong = lowerLong + randomIntBetween(0, 32);
    addQuery(LongPoint.newRangeQuery("long_field", lowerLong, upperLong), documents);

    indexWriter.addDocuments(documents);
    indexWriter.close();
    directoryReader = DirectoryReader.open(directory);
    IndexSearcher shardSearcher = newSearcher(directoryReader);
    // Disable query cache, because ControlQuery cannot be cached...
    shardSearcher.setQueryCache(null);

    Document document = new Document();
    document.add(new TextField("field", "the quick brown fox jumps over the lazy dog", Field.Store.NO));
    long randomLong = randomIntBetween((int) lowerLong, (int) upperLong);
    document.add(new LongPoint("long_field", randomLong));
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());
    duelRun(queryStore, memoryIndex, shardSearcher);
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testCommonTermsQueryHighlight() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testCommonTermsQueryHighlightTest() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,  new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = new IndexSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}
 

import org.apache.lucene.queries.CommonTermsQuery; //导入方法依赖的package包/类
public void testCommonTermsQueryHighlightTest() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,  new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
  FieldType type = new FieldType(TextField.TYPE_STORED);
  type.setStoreTermVectorOffsets(true);
  type.setStoreTermVectorPositions(true);
  type.setStoreTermVectors(true);
  type.freeze();
  String[] texts = {
      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
      "JFK has been shot", "John Kennedy has been shot",
      "This text has a typo in referring to Keneddy",
      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
  for (int i = 0; i < texts.length; i++) {
    Document doc = new Document();
    Field field = new Field("field", texts[i], type);
    doc.add(field);
    writer.addDocument(doc);
  }
  CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 2);
  query.add(new Term("field", "text"));
  query.add(new Term("field", "long"));
  query.add(new Term("field", "very"));
 
  FastVectorHighlighter highlighter = new FastVectorHighlighter();
  IndexReader reader = DirectoryReader.open(writer, true);
  IndexSearcher searcher = newSearcher(reader);
  TopDocs hits = searcher.search(query, 10);
  assertEquals(2, hits.totalHits);
  FieldQuery fieldQuery  = highlighter.getFieldQuery(query, reader);
  String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
  assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);

  fieldQuery  = highlighter.getFieldQuery(query, reader);
  bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
  assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);

  reader.close();
  writer.close();
  dir.close();
}