Java TFIDFSimilarity类代码示例

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
private void assertScoresMatch(List<PrebuiltFeature> features, float[] scores,
                               RankerQuery ltrQuery, ScoreDoc scoreDoc) throws IOException {
    Document d = searcherUnderTest.doc(scoreDoc.doc);
    String idVal = d.get("id");
    int docId = Integer.decode(idVal);
    float modelScore = scores[docId];
    float queryScore = scoreDoc.score;

    assertEquals("Scores match with similarity " + similarity.getClass(), modelScore,
            queryScore, SCORE_NB_ULP_PREC *Math.ulp(modelScore));

    if (!(similarity instanceof TFIDFSimilarity)) {
        // There are precision issues with these similarities when using explain
        // It produces 0.56103003 for feat:0 in doc1 using score() but 0.5610301 using explain
        Explanation expl = searcherUnderTest.explain(ltrQuery, docId);

        assertEquals("Explain scores match with similarity " + similarity.getClass(), expl.getValue(),
                queryScore, 5 * Math.ulp(modelScore));
        checkFeatureNames(expl, features);
    }
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  IndexSearcher searcher = (IndexSearcher)context.get("searcher");
  final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), field);
  if (similarity == null) {
    throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
  }
  final NumericDocValues norms = readerContext.reader().getNormValues(field);

  if (norms == null) {
    return new ConstDoubleDocValues(0.0, this);
  }
  
  return new FloatDocValues(this) {
    @Override
    public float floatVal(int doc) {
      return similarity.decodeNormValue(norms.get(doc));
    }
  };
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void testHyperbolicSweetSpot() {

  SweetSpotSimilarity ss = new SweetSpotSimilarity() {
      @Override
      public float tf(float freq) {
        return hyperbolicTf(freq);
      }
    };
  ss.setHyperbolicTfFactors(3.3f, 7.7f, Math.E, 5.0f);
  
  TFIDFSimilarity s = ss;

  for (int i = 1; i <=1000; i++) {
    assertTrue("MIN tf: i="+i+" : s="+s.tf(i),
               3.3f <= s.tf(i));
    assertTrue("MAX tf: i="+i+" : s="+s.tf(i),
               s.tf(i) <= 7.7f);
  }
  assertEquals("MID tf", 3.3f+(7.7f - 3.3f)/2.0f, s.tf(5), 0.00001f);
  
  // stupidity
  assertEquals("tf zero", 0.0f, s.tf(0), 0.0f);
  
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  IndexSearcher searcher = (IndexSearcher)context.get("searcher");
  final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), field);
  if (similarity == null) {
    throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
  }
  final NumericDocValues norms = readerContext.reader().getNormValues(field);

  if (norms == null) {
    return new ConstDoubleDocValues(0.0, this);
  }
  
  return new FloatDocValues(this) {
    @Override
    public float floatVal(int doc) {
      return similarity.decodeNormValue((byte)norms.get(doc));
    }
  };
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void testHyperbolicSweetSpot() {

  SweetSpotSimilarity ss = new SweetSpotSimilarity() {
      @Override
      public float tf(int freq) {
        return hyperbolicTf(freq);
      }
    };
  ss.setHyperbolicTfFactors(3.3f, 7.7f, Math.E, 5.0f);
  
  TFIDFSimilarity s = ss;

  for (int i = 1; i <=1000; i++) {
    assertTrue("MIN tf: i="+i+" : s="+s.tf(i),
               3.3f <= s.tf(i));
    assertTrue("MAX tf: i="+i+" : s="+s.tf(i),
               s.tf(i) <= 7.7f);
  }
  assertEquals("MID tf", 3.3f+(7.7f - 3.3f)/2.0f, s.tf(5), 0.00001f);
  
  // stupidity
  assertEquals("tf zero", 0.0f, s.tf(0), 0.0f);
  
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  IndexSearcher searcher = (IndexSearcher)context.get("searcher");
  TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(), field);
  if (sim == null) {
    throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
  }
  int docfreq = searcher.getIndexReader().docFreq(new Term(indexedField, indexedBytes));
  float idf = sim.idf(docfreq, searcher.getIndexReader().maxDoc());
  return new ConstDoubleDocValues(idf, this);
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
static TFIDFSimilarity asTFIDF(Similarity sim, String field) {
  while (sim instanceof PerFieldSimilarityWrapper) {
    sim = ((PerFieldSimilarityWrapper)sim).get(field);
  }
  if (sim instanceof TFIDFSimilarity) {
    return (TFIDFSimilarity)sim;
  } else {
    return null;
  }
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void testSimple2() throws Exception {
  assumeTrue("Broken scoring: LUCENE-3723", 
      searcher.getSimilarity() instanceof TFIDFSimilarity);
  SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
  SpanQuery q2 = new SpanTermQuery(new Term("last", "smith"));
  SpanQuery q = new SpanNearQuery(new SpanQuery[]
    { q1, new FieldMaskingSpanQuery(q2, "gender")}, -1, false );
  check(q, new int[] { 2, 4 });
  q = new SpanNearQuery(new SpanQuery[]
    { new FieldMaskingSpanQuery(q1, "id"),
      new FieldMaskingSpanQuery(q2, "id") }, -1, false );
  check(q, new int[] { 2, 4 });
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void testSpans2() throws Exception {
  assumeTrue("Broken scoring: LUCENE-3723", 
      searcher.getSimilarity() instanceof TFIDFSimilarity);
  SpanQuery qA1 = new SpanTermQuery(new Term("gender", "female"));
  SpanQuery qA2 = new SpanTermQuery(new Term("first",  "james"));
  SpanQuery qA  = new SpanOrQuery(qA1, new FieldMaskingSpanQuery(qA2, "gender"));
  SpanQuery qB  = new SpanTermQuery(new Term("last",   "jones"));
  SpanQuery q   = new SpanNearQuery(new SpanQuery[]
    { new FieldMaskingSpanQuery(qA, "id"),
      new FieldMaskingSpanQuery(qB, "id") }, -1, false );
  check(q, new int[] { 0, 1, 2, 3 });

  Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
  
  assertEquals(true, span.next());
  assertEquals(s(0,0,1), s(span));

  assertEquals(true, span.next());
  assertEquals(s(1,1,2), s(span));

  assertEquals(true, span.next());
  assertEquals(s(2,0,1), s(span));

  assertEquals(true, span.next());
  assertEquals(s(2,2,3), s(span));

  assertEquals(true, span.next());
  assertEquals(s(3,0,1), s(span));

  assertEquals(false, span.next());
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public AbstractAuthorityQueryWeight(SolrIndexSearcher searcher, Query query, String authTermName, String authTermText) throws IOException
{
    this.searcher = searcher;
    this.query = query;
    this.similarity = (TFIDFSimilarity) searcher.getSimilarity();
    CollectionStatistics collectionStats = searcher.collectionStatistics(authTermName);
    final IndexReaderContext context = searcher.getTopReaderContext();
    final Term term = new Term(authTermName, authTermText);
    final TermContext termContext = TermContext.build(context, term);
    TermStatistics termStats = searcher.termStatistics(term, termContext);
    idfExp = similarity.idfExplain(collectionStats, termStats);
    idf = idfExp.getValue();
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void setSimilarity(Similarity similarity) {
    if (similarity == null || similarity instanceof TFIDFSimilarity) {
        //LUCENE 4 UPGRADE we need TFIDF similarity here so I only set it if it is an instance of it
        this.similarity = (TFIDFSimilarity) similarity;
    }
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public XMoreLikeThis(IndexReader ir, TFIDFSimilarity sim) {
    this.ir = ir;
    this.similarity = sim;
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public TFIDFSimilarity getSimilarity() {
    return similarity;
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public void setSimilarity(TFIDFSimilarity similarity) {
    this.similarity = similarity;
}
 

import org.apache.lucene.search.similarities.TFIDFSimilarity; //导入依赖的package包/类
public RelevancyFeedback(IndexReader ir, TFIDFSimilarity sim) {
    this.ir = ir;
    this.similarity = sim;

}