Java TermStatistics类代码示例

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public void writeTo(final StreamOutput out) throws IOException {
    out.writeVInt(termStatistics.size());

    for (ObjectObjectCursor<Term, TermStatistics> c : termStatistics()) {
        Term term = c.key;
        out.writeString(term.field());
        out.writeBytesRef(term.bytes());
        TermStatistics stats = c.value;
        out.writeBytesRef(stats.term());
        out.writeVLong(stats.docFreq());
        out.writeVLong(DfsSearchResult.addOne(stats.totalTermFreq()));
    }

    DfsSearchResult.writeFieldStats(out, fieldStatistics);
    out.writeVLong(maxDoc);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
public static TermStatistics[] readTermStats(StreamInput in, Term[] terms) throws IOException {
    int termsStatsSize = in.readVInt();
    final TermStatistics[] termStatistics;
    if (termsStatsSize == 0) {
        termStatistics = EMPTY_TERM_STATS;
    } else {
        termStatistics = new TermStatistics[termsStatsSize];
        assert terms.length == termsStatsSize;
        for (int i = 0; i < termStatistics.length; i++) {
            BytesRef term = terms[i].bytes();
            final long docFreq = in.readVLong();
            assert docFreq >= 0;
            final long totalTermFreq = subOne(in.readVLong());
            termStatistics[i] = new TermStatistics(term, docFreq, totalTermFreq);
        }
    }
    return termStatistics;
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
protected SeqSpanWeight(SeqSpanQuery query, IndexSearcher searcher) throws IOException {
  super(query);
  this.selfQuery = query;
  this.similarity = searcher.getSimilarity(needsScores);
  this.positions = selfQuery.getPositions();
  this.terms = selfQuery.getTerms();
  this.field = terms[0].field();
  if (positions.length < 2) {
    throw new IllegalStateException("PhraseWeight does not support less than 2 terms, call rewrite first");
  } else if (positions[0] != 0) {
    throw new IllegalStateException("PhraseWeight requires that the first position is 0, call rewrite first");
  }
  final IndexReaderContext context = searcher.getTopReaderContext();
  states = new TermContext[terms.length];
  TermStatistics termStats[] = new TermStatistics[terms.length];
  for (int i = 0; i < terms.length; i++) {
    final Term term = terms[i];
    states[i] = TermContext.build(context, term);
    termStats[i] = searcher.termStatistics(term, states[i]);
  }
  stats = similarity.computeWeight(searcher.collectionStatistics(terms[0].field()), termStats);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public void writeTo(final StreamOutput out) throws IOException {
    out.writeVInt(termStatistics.size());
    
    for (ObjectObjectCursor<Term, TermStatistics> c : termStatistics()) {
        Term term = (Term) c.key;
        out.writeString(term.field());
        out.writeBytesRef(term.bytes());
        TermStatistics stats = (TermStatistics) c.value;
        out.writeBytesRef(stats.term());
        out.writeVLong(stats.docFreq());
        out.writeVLong(DfsSearchResult.addOne(stats.totalTermFreq()));
    }

    DfsSearchResult.writeFieldStats(out, fieldStatistics);
    out.writeVLong(maxDoc);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
   public final SimWeight computeWeight(CollectionStatistics collectionStats,
				 TermStatistics... termStats)
   {
float N, n, idf, adl;
idf = 1.0f;
N   = collectionStats.maxDoc();
adl = collectionStats.sumTotalTermFreq() / N;

if (termStats.length == 1) {
    n = termStats[0].docFreq();
    idf = log(N/n);
}
else {
    for (final TermStatistics stat : termStats) {
	n = stat.docFreq();
	idf += log(N/n);
    }
}

return new TFIDFWeight(collectionStats.field(), idf, adl);
   }
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
/**
 * Computes a score factor for a phrase.
 * 
 * <p>
 * The default implementation sums the idf factor for each term in the
 * phrase.
 * 
 * @param collectionStats
 *            collection-level statistics
 * @param termStats
 *            term-level statistics for the terms in the phrase
 * @return an Explain object that includes both an idf score factor for the
 *         phrase and an explanation for each term.
 */
public Explanation idfExplain(final CollectionStatistics collectionStats,
		final TermStatistics termStats[]) {
	final long max = collectionStats.maxDoc();
	float idf = 0.0f;
	final Explanation exp = new Explanation();
	exp.setDescription("idf(), sum of:");
	for (final TermStatistics stat : termStats) {
		final long docFreq = stat.docFreq();
		final float termIdf = idf(docFreq, max);
		exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + docFreq
				+ ", maxDocs=" + max + ")"));
		idf += termIdf;
	}
	exp.setValue(idf);
	return exp;
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public void readFrom(StreamInput in) throws IOException {
    int size = in.readVInt();
    termStatistics = HppcMaps.newMap(size);
    for (int i = 0; i < size; i++) {
        Term term = new Term(in.readString(), in.readBytesRef());
        TermStatistics stats = new TermStatistics(in.readBytesRef(),
                in.readVLong(),
                DfsSearchResult.subOne(in.readVLong()));
        termStatistics.put(term, stats);
    }
    fieldStatistics = DfsSearchResult.readFieldStats(in);
    maxDoc = in.readVLong();
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
private void writeTermStatistics(TermStatistics termStatistics) throws IOException {
    int docFreq = (int) termStatistics.docFreq();
    assert (docFreq >= -1);
    writePotentiallyNegativeVInt(docFreq);
    long ttf = termStatistics.totalTermFreq();
    assert (ttf >= -1);
    writePotentiallyNegativeVLong(ttf);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
public void testFailPhaseOnException() throws IOException {
    AtomicArray<DfsSearchResult> results = new AtomicArray<>(2);
    AtomicReference<AtomicArray<QuerySearchResultProvider>> responseRef = new AtomicReference<>();
    results.set(0, new DfsSearchResult(1, new SearchShardTarget("node1", new Index("test", "na"), 0)));
    results.set(1, new DfsSearchResult(2, new SearchShardTarget("node2", new Index("test", "na"), 0)));
    results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
    results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);

    SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
    SearchTransportService searchTransportService = new SearchTransportService(
        Settings.builder().put("search.remote.connect", false).build(), null,  null) {

        @Override
        public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                     ActionListener<QuerySearchResult> listener) {
            if (request.id() == 1) {
                QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new Index("test", "na"), 0));
                queryResult.topDocs(new TopDocs(1, new ScoreDoc[] {new ScoreDoc(42, 1.0F)}, 2.0F), new DocValueFormat[0]);
                queryResult.size(2); // the size of the result set
                listener.onResponse(queryResult);
            } else if (request.id() == 2) {
               throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
            } else {
                fail("no such request ID: " + request.id());
            }
        }
    };
    MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
    mockSearchPhaseContext.searchTransport = searchTransportService;
    DfsQueryPhase phase = new DfsQueryPhase(results, controller,
        (response) -> new SearchPhase("test") {
            @Override
            public void run() throws IOException {
                responseRef.set(response.results);
            }
        }, mockSearchPhaseContext);
    assertEquals("dfs_query", phase.getName());
    expectThrows(UncheckedIOException.class, () -> phase.run());
    assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty()); // phase execution will clean up on the contexts
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public final SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
  BasicStats stats[] = new BasicStats[termStats.length];
  for (int i = 0; i < termStats.length; i++) {
    stats[i] = newStats(collectionStats.field(), queryBoost);
    fillBasicStats(stats[i], collectionStats, termStats[i]);
  }
  return stats.length == 1 ? stats[0] : new MultiSimilarity.MultiStats(stats);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
/** Fills all member fields defined in {@code BasicStats} in {@code stats}. 
 *  Subclasses can override this method to fill additional stats. */
protected void fillBasicStats(BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
  // #positions(field) must be >= #positions(term)
  assert collectionStats.sumTotalTermFreq() == -1 || collectionStats.sumTotalTermFreq() >= termStats.totalTermFreq();
  long numberOfDocuments = collectionStats.maxDoc();
  
  long docFreq = termStats.docFreq();
  long totalTermFreq = termStats.totalTermFreq();

  // codec does not supply totalTermFreq: substitute docFreq
  if (totalTermFreq == -1) {
    totalTermFreq = docFreq;
  }

  final long numberOfFieldTokens;
  final float avgFieldLength;

  long sumTotalTermFreq = collectionStats.sumTotalTermFreq();

  if (sumTotalTermFreq <= 0) {
    // field does not exist;
    // We have to provide something if codec doesnt supply these measures,
    // or if someone omitted frequencies for the field... negative values cause
    // NaN/Inf for some scorers.
    numberOfFieldTokens = docFreq;
    avgFieldLength = 1;
  } else {
    numberOfFieldTokens = sumTotalTermFreq;
    avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments;
  }
 
  // TODO: add sumDocFreq for field (numberOfFieldPostings)
  stats.setNumberOfDocuments(numberOfDocuments);
  stats.setNumberOfFieldTokens(numberOfFieldTokens);
  stats.setAvgFieldLength(avgFieldLength);
  stats.setDocFreq(docFreq);
  stats.setTotalTermFreq(totalTermFreq);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
/**
 * Computes a score factor for a phrase.
 * 
 * <p>
 * The default implementation sums the idf factor for
 * each term in the phrase.
 * 
 * @param collectionStats collection-level statistics
 * @param termStats term-level statistics for the terms in the phrase
 * @return an Explain object that includes both an idf 
 *         score factor for the phrase and an explanation 
 *         for each term.
 */
public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
  final long max = collectionStats.maxDoc();
  float idf = 0.0f;
  final Explanation exp = new Explanation();
  exp.setDescription("idf(), sum of:");
  for (final TermStatistics stat : termStats ) {
    final long df = stat.docFreq();
    final float termIdf = idf(df, max);
    exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"));
    idf += termIdf;
  }
  exp.setValue(idf);
  return exp;
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public final SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
  Explanation idf = termStats.length == 1 ? idfExplain(collectionStats, termStats[0]) : idfExplain(collectionStats, termStats);

  float avgdl = avgFieldLength(collectionStats);

  // compute freq-independent part of bm25 equation across all norm values
  float cache[] = new float[256];
  for (int i = 0; i < cache.length; i++) {
    cache[i] = k1 * ((1 - b) + b * decodeNormValue((byte)i) / avgdl);
  }
  return new BM25Stats(collectionStats.field(), idf, queryBoost, avgdl, cache);
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public final SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
  PerFieldSimWeight weight = new PerFieldSimWeight();
  weight.delegate = get(collectionStats.field());
  weight.delegateWeight = weight.delegate.computeWeight(queryBoost, collectionStats, termStats);
  return weight;
}
 

import org.apache.lucene.search.TermStatistics; //导入依赖的package包/类
@Override
public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
  SimWeight subStats[] = new SimWeight[sims.length];
  for (int i = 0; i < subStats.length; i++) {
    subStats[i] = sims[i].computeWeight(queryBoost, collectionStats, termStats);
  }
  return new MultiStats(subStats);
}