Java BM25Similarity类代码示例

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
public CollectiveTestApproach(boolean fuzzy, boolean standardSeacher,
		boolean withDescription) {
	File indexDir = new File(entIndexDirectory);
	File indexDir1 = new File(docIndexDirectory);
	this.fuzzy = fuzzy;
	this.withDescription = withDescription;
	try {
		Directory dir = FSDirectory.open(indexDir);
		Directory dir1 = FSDirectory.open(indexDir1);
		entISearcher = new IndexSearcher(DirectoryReader.open(dir));
		entIReader = DirectoryReader.open(dir);
		docISearcher = new IndexSearcher(DirectoryReader.open(dir1));
		docIReader = DirectoryReader.open(dir1);
		if (!standardSeacher) {
			entISearcher.setSimilarity(new BM25Similarity());
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
public BM25SimilarityProvider(String name, Settings settings, Settings indexSettings) {
    super(name);
    float k1 = settings.getAsFloat("k1", 1.2f);
    float b = settings.getAsFloat("b", 0.75f);
    final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(getClass()));
    boolean discountOverlaps =
        settings.getAsBooleanLenientForPreEs6Indices(Version.indexCreated(indexSettings), "discount_overlaps", true, deprecationLogger);

    this.similarity = new BM25Similarity(k1, b);
    this.similarity.setDiscountOverlaps(discountOverlaps);
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
@Inject
public BM25SimilarityProvider(@Assisted String name, @Assisted Settings settings) {
    super(name);
    float k1 = settings.getAsFloat("k1", 1.2f);
    float b = settings.getAsFloat("b", 0.75f);
    boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);

    this.similarity = new BM25Similarity(k1, b);
    this.similarity.setDiscountOverlaps(discountOverlaps);
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
private void openReader() throws IOException {
    this.hitsReader   = DirectoryReader.open(FSDirectory.open(this.hitsDirectory));
    this.hitsSearcher = new IndexSearcher(this.hitsReader);
    this.hitsSearcher.setSimilarity(new BM25Similarity(0.0f, 0.0f)); // simple idf scoring 
    //searcher.setSimilarity(new BM25Similarity(1.2f, 0.75f)); // k1, b
    //searcher.setSimilarity(new LMDirichletSimilarity(200f)); // mu
    //searcher.setSimilarity(new LMJelinekMercerSimilarity(0.5f)); // lambda
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
@Before
public void setupIndex() throws IOException {
    dirUnderTest = newDirectory();
    List<Similarity> sims = Arrays.asList(
            new ClassicSimilarity(),
            new SweetSpotSimilarity(), // extends Classic
            new BM25Similarity(),
            new LMDirichletSimilarity(),
            new BooleanSimilarity(),
            new LMJelinekMercerSimilarity(0.2F),
            new AxiomaticF3LOG(0.5F, 10),
            new DFISimilarity(new IndependenceChiSquared()),
            new DFRSimilarity(new BasicModelBE(), new AfterEffectB(), new NormalizationH1()),
            new IBSimilarity(new DistributionLL(), new LambdaDF(), new NormalizationH3())
        );
    similarity = sims.get(random().nextInt(sims.size()));

    indexWriterUnderTest = new RandomIndexWriter(random(), dirUnderTest, newIndexWriterConfig().setSimilarity(similarity));
    for (int i = 0; i < docs.length; i++) {
        Document doc = new Document();
        doc.add(newStringField("id", "" + i, Field.Store.YES));
        doc.add(newField("field", docs[i], Store.YES));
        indexWriterUnderTest.addDocument(doc);
    }
    indexWriterUnderTest.commit();
    indexWriterUnderTest.forceMerge(1);
    indexWriterUnderTest.flush();


    indexReaderUnderTest = indexWriterUnderTest.getReader();
    searcherUnderTest = newSearcher(indexReaderUnderTest);
    searcherUnderTest.setSimilarity(similarity);
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
public LuceneReRank(String indexLocation) throws IOException {
    dir = FSDirectory.open(new File(indexLocation));
    IndexWriterConfig iwc = new IndexWriterConfig(ANALYZER.getVersion(), ANALYZER);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    writer = new IndexWriter(dir, iwc);
    float K1 = (float) 1.0;
    float B = (float) 0.75;
    //NOTE: Leo mentioned that lucene's bm25 calculation could be not accurate
    similarity = new BM25Similarity(K1, B);
    fieldToLoad.add(RE_RANK_OFFSET);
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/** bm25 with parameters */
public void testParameters() throws Exception {
  Similarity sim = getSimilarity("text_params");
  assertEquals(BM25Similarity.class, sim.getClass());
  BM25Similarity bm25 = (BM25Similarity) sim;
  assertEquals(1.2f, bm25.getK1(), 0.01f);
  assertEquals(0.76f, bm25.getB(), 0.01f);
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
@JsonIgnore
public Similarity getScoringAlgorithmObject() {
    if(this.scoringAlgorithm == null || this.scoringAlgorithm.isEmpty() || this.scoringAlgorithm.equals(DEFAULT_SCORING_ALGORITHM) || this.scoringAlgorithm.equalsIgnoreCase("tfidf") || this.scoringAlgorithm.equalsIgnoreCase("vectorspace")) {
        // vector-space model
        return new DefaultSimilarity();
    } else if(this.scoringAlgorithm.equalsIgnoreCase("bm25")) {
        // bm25 probability model
        return new BM25Similarity();
    }
    
    return new DefaultSimilarity();
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
 * Feature 5: cos(BM25) * sim(t_d, q)
 * 
 * @param keyword
 * @return
 */
private Query createFeature5(EntityObject dataObject) {
	String keyword = dataObject.getText();
	BM25Similarity bm25 = new BM25Similarity();
	LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term("title",
			keyword), bm25);
	return fq;
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
 * Feature 6: cos(Bm25) * sim(a_d, q)
 * 
 * @param dataObject
 * @return
 */
private Query createFeature6(EntityObject dataObject) {
	String keyword = dataObject.getText();
	BM25Similarity bm25 = new BM25Similarity();
	LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
			"description", keyword), bm25);
	return fq;
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
	 * Feature 7: cos(BM25) * sim(t_d, q_c)
	 * 
	 * @param dataObject
	 * @return
	 */
	private Query createFeature7(EntityObject dataObject) {
		String sentence = dataObject.getContext();
		String[] split = sentence.split(" ");
		LTRBooleanQuery bq = new LTRBooleanQuery();
		BM25Similarity bm25 = new BM25Similarity();
		for (int i = 0; i < split.length; i++) {
//			LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
//					"title", split[i]), bm25);
			 LearnToRankTermQuery fq = new LearnToRankTermQuery(new Term(
			 "title", split[i]), bm25);
			bq.add(fq, Occur.SHOULD);
		}
		return bq;
	}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
	 * Feature 8: cos(BM25) * sim(a_d, q_c)
	 * 
	 * @param dataObject
	 * @return
	 */
	private Query createFeature8(EntityObject dataObject) {
		String sentence = dataObject.getContext();
		String[] split = sentence.split(" ");
		LTRBooleanQuery bq = new LTRBooleanQuery();
		BM25Similarity bm25 = new BM25Similarity();
		for (int i = 0; i < split.length; i++) {
//			LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
//					"description", usePorterStemmer(split[i])), bm25);
			 LearnToRankTermQuery fq = new LearnToRankTermQuery(new Term(
			 "description", split[i]), bm25);
			bq.add(fq, Occur.SHOULD);
		}
		return bq;
	}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
 * Feature 6: cos(Bm25) * sim(a_d, q)
 * 
 * @param dataObject
 * @return
 */
private Query createFeature6(EntityObject dataObject) {
	String keyword = dataObject.getText();
	BM25Similarity bm25 = new BM25Similarity();
	LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
			"abstract", keyword), bm25);
	return fq;
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
 * Feature 7: cos(BM25) * sim(t_d, q_c)
 * 
 * @param dataObject
 * @return
 */
private Query createFeature7(EntityObject dataObject) {
	String sentence = dataObject.getContext();
	String[] split = sentence.split(" ");
	LTRBooleanQuery bq = new LTRBooleanQuery();
	BM25Similarity bm25 = new BM25Similarity();
	for (int i = 0; i < split.length; i++) {
		LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
				"title", split[i]), bm25);
		bq.add(fq, Occur.SHOULD);
	}
	return bq;
}
 

import org.apache.lucene.search.similarities.BM25Similarity; //导入依赖的package包/类
/**
 * Feature 8: cos(BM25) * sim(a_d, q_c)
 * 
 * @param dataObject
 * @return
 */
private Query createFeature8(EntityObject dataObject) {
	String sentence = dataObject.getContext();
	String[] split = sentence.split(" ");
	LTRBooleanQuery bq = new LTRBooleanQuery();
	BM25Similarity bm25 = new BM25Similarity();
	for (int i = 0; i < split.length; i++) {
		LearnToRankFuzzyQuery fq = new LearnToRankFuzzyQuery(new Term(
				"abstract", split[i]), bm25);
		bq.add(fq, Occur.SHOULD);
	}
	return bq;
}