Java Similarity类代码示例

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
SeqSpanScorer(SeqSpanWeight weight, PostingsAndFreq[] postings,
                  Similarity.SimScorer docScorer, boolean needsScores,
                  float matchCost) throws IOException {
  super(weight);
  this.selfWeight = weight;
  this.docScorer = docScorer;
  this.needsScores = needsScores;

  List<DocIdSetIterator> iterators = new ArrayList<>();
  List<PostingsAndPosition> postingsAndPositions = new ArrayList<>();
  for(PostingsAndFreq posting : postings) {
    iterators.add(posting.postings);
    postingsAndPositions.add(new PostingsAndPosition(posting.postings, posting.position));
  }
  conjunction = ConjunctionDISI.intersectIterators(iterators);
  this.postings = postingsAndPositions.toArray(new PostingsAndPosition[postingsAndPositions.size()]);
  this.matchCost = matchCost;
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
                  Similarity.SimScorer docScorer) throws IOException {
  super(weight);
  this.docScorer = docScorer;

  chunkStates = new ChunkState[postings.length];

  endMinus1 = postings.length-1;
  
  lead = postings[0].postings;
  // min(cost)
  cost = lead.cost();

  for(int i=0;i<postings.length;i++) {
    chunkStates[i] = new ChunkState(postings[i].postings, -postings[i].position);
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
protected void processPayload(Similarity similarity) throws IOException {
  if (termSpans.isPayloadAvailable()) {
    final DocsAndPositionsEnum postings = termSpans.getPostings();
    payload = postings.getPayload();
    if (payload != null) {
      payloadScore = function.currentScore(doc, term.field(),
                                           spans.start(), spans.end(), payloadsSeen, payloadScore,
                                           docScorer.computePayloadFactor(doc, spans.start(), spans.end(), payload));
    } else {
      payloadScore = function.currentScore(doc, term.field(),
                                           spans.start(), spans.end(), payloadsSeen, payloadScore, 1F);
    }
    payloadsSeen++;

  } else {
    // zero out the payload?
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
@Override
public NumericDocValues getNormValues(String field) {
  FieldInfo fieldInfo = fieldInfos.get(field);
  if (fieldInfo == null || fieldInfo.omitsNorms())
    return null;
  NumericDocValues norms = cachedNormValues;
  Similarity sim = getSimilarity();
  if (!field.equals(cachedFieldName) || sim != cachedSimilarity) { // not cached?
    Info info = getInfo(field);
    int numTokens = info != null ? info.numTokens : 0;
    int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
    float boost = info != null ? info.getBoost() : 1.0f; 
    FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
    long value = sim.computeNorm(invertState);
    norms = new MemoryIndexNormDocValues(value);
    // cache it for future reuse
    cachedNormValues = norms;
    cachedFieldName = field;
    cachedSimilarity = sim;
    if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
  }
  return norms;
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
public void testReallyNoNormsForDrillDown() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setSimilarity(new PerFieldSimilarityWrapper() {
      final Similarity sim = new DefaultSimilarity();

      @Override
      public Similarity get(String name) {
        assertEquals("field", name);
        return sim;
      }
    });
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  writer.addDocument(config.build(taxoWriter, doc));
  IOUtils.close(writer, taxoWriter, dir, taxoDir);
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
/**
 * Sets up a RAMDirectory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new RAMDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      TEST_VERSION_CURRENT, analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  reader = DirectoryReader.open(writer, true);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(reader);
  searcher.setSimilarity(similarity);
  return searcher;
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
public void testRewriteCoord1() throws Exception {
  final Similarity oldSimilarity = s.getSimilarity();
  try {
    s.setSimilarity(new DefaultSimilarity() {
      @Override
      public float coord(int overlap, int maxOverlap) {
        return overlap / ((float)maxOverlap + 1);
      }
    });
    BooleanQuery q1 = new BooleanQuery();
    q1.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
    BooleanQuery q2 = new BooleanQuery();
    q2.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
    q2.setMinimumNumberShouldMatch(1);
    TopDocs top1 = s.search(q1,null,100);
    TopDocs top2 = s.search(q2,null,100);
    assertSubsetOfSameScores(q2, top1, top2);
  } finally {
    s.setSimilarity(oldSimilarity);
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
public void testRewriteNegate() throws Exception {
  final Similarity oldSimilarity = s.getSimilarity();
  try {
    s.setSimilarity(new DefaultSimilarity() {
      @Override
      public float coord(int overlap, int maxOverlap) {
        return overlap / ((float)maxOverlap + 1);
      }
    });
    BooleanQuery q1 = new BooleanQuery();
    q1.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
    BooleanQuery q2 = new BooleanQuery();
    q2.add(new TermQuery(new Term("data", "1")), BooleanClause.Occur.SHOULD);
    q2.add(new TermQuery(new Term("data", "Z")), BooleanClause.Occur.MUST_NOT);
    TopDocs top1 = s.search(q1,null,100);
    TopDocs top2 = s.search(q2,null,100);
    assertSubsetOfSameScores(q2, top1, top2);
  } finally {
    s.setSimilarity(oldSimilarity);
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
@Test
public void testQueries10() throws Exception {
  BooleanQuery query = new BooleanQuery();
  query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
  query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
  query.add(new TermQuery(new Term(field, "w2")), BooleanClause.Occur.MUST);
  query.add(new TermQuery(new Term(field, "zz")), BooleanClause.Occur.SHOULD);

  int[] expDocNrs = {2, 3};
  Similarity oldSimilarity = searcher.getSimilarity();
  try {
    searcher.setSimilarity(new DefaultSimilarity(){
      @Override
      public float coord(int overlap, int maxOverlap) {
        return overlap / ((float)maxOverlap - 1);
      }
    });
    queriesTest(query, expDocNrs);
  } finally {
    searcher.setSimilarity(oldSimilarity);
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
public void buildIndex(Directory dir) throws IOException {
  Random random = random();
  MockAnalyzer analyzer = new MockAnalyzer(random());
  analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
  IndexWriterConfig config = newIndexWriterConfig(analyzer);
  Similarity provider = new MySimProvider();
  config.setSimilarity(provider);
  RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
  final LineFileDocs docs = new LineFileDocs(random, defaultCodecSupportsDocValues());
  int num = atLeast(100);
  for (int i = 0; i < num; i++) {
    Document doc = docs.nextDoc();
    int boost = random().nextInt(255);
    Field f = new TextField(byteTestField, "" + boost, Field.Store.YES);
    f.setBoost(boost);
    doc.add(f);
    writer.addDocument(doc);
    doc.removeField(byteTestField);
    if (rarely()) {
      writer.commit();
    }
  }
  writer.commit();
  writer.close();
  docs.close();
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) {
  if (node==null) {
    return null;
  } else {
    SimilarityFactory similarityFactory;
    final String classArg = ((Element) node).getAttribute(SimilarityFactory.CLASS_NAME);
    final Object obj = loader.newInstance(classArg, Object.class, "search.similarities.");
    if (obj instanceof SimilarityFactory) {
      // configure a factory, get a similarity back
      final NamedList<Object> namedList = DOMUtil.childNodesToNamedList(node);
      namedList.add(SimilarityFactory.CLASS_NAME, classArg);
      SolrParams params = SolrParams.toSolrParams(namedList);
      similarityFactory = (SimilarityFactory)obj;
      similarityFactory.init(params);
    } else {
      // just like always, assume it's a Similarity and get a ClassCastException - reasonable error handling
      similarityFactory = new SimilarityFactory() {
        @Override
        public Similarity getSimilarity() {
          return (Similarity) obj;
        }
      };
    }
    return similarityFactory;
  }
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
private void initialize(File indexPath, int kmerSize, int kmerSkips, boolean minStrandKmer, double minShouldMatch, QueryGenerationAlgorithm queryGenerationAlgorithm, Similarity similarity) throws Exception {
    if(!indexPath.exists() || !indexPath.isDirectory()) {
        throw new IllegalArgumentException("indexPath is not a directory or does not exist");
    }
    
    this.indexPath = indexPath;
    this.kmerSize = kmerSize;
    this.kmerSkips = kmerSkips;
    this.minStrandKmer = minStrandKmer;
    this.queryAnalyzer = new KmerQueryAnalyzer(this.kmerSize, this.kmerSkips, this.minStrandKmer);
    Directory dir = new MMapDirectory(this.indexPath.toPath()); 
    this.indexReader = DirectoryReader.open(dir);
    this.indexSearcher = new IndexSearcher(this.indexReader);
    if(similarity != null) {
        this.indexSearcher.setSimilarity(similarity);
    }
    this.minShouldMatch = minShouldMatch;
    this.queryGenerationAlgorithm = queryGenerationAlgorithm;
    
    BooleanQuery.setMaxClauseCount(10000);
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
LateParsingQuery(Query toQuery, Query innerQuery, int minChildren, int maxChildren,
                 String parentType, ScoreMode scoreMode, ParentChildIndexFieldData parentChildIndexFieldData,
                 Similarity similarity) {
    this.toQuery = toQuery;
    this.innerQuery = innerQuery;
    this.minChildren = minChildren;
    this.maxChildren = maxChildren;
    this.parentType = parentType;
    this.scoreMode = scoreMode;
    this.parentChildIndexFieldData = parentChildIndexFieldData;
    this.similarity = similarity;
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
/**
 * Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
 */
public EngineConfig(OpenMode openMode, ShardId shardId, ThreadPool threadPool,
                    IndexSettings indexSettings, Engine.Warmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
                    MergePolicy mergePolicy, Analyzer analyzer,
                    Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
                    TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
                    TranslogConfig translogConfig, TimeValue flushMergesAfter, ReferenceManager.RefreshListener refreshListeners,
                    long maxUnsafeAutoIdTimestamp) {
    if (openMode == null) {
        throw new IllegalArgumentException("openMode must not be null");
    }
    this.shardId = shardId;
    this.indexSettings = indexSettings;
    this.threadPool = threadPool;
    this.warmer = warmer == null ? (a) -> {} : warmer;
    this.store = store;
    this.deletionPolicy = deletionPolicy;
    this.mergePolicy = mergePolicy;
    this.analyzer = analyzer;
    this.similarity = similarity;
    this.codecService = codecService;
    this.eventListener = eventListener;
    codecName = indexSettings.getValue(INDEX_CODEC_SETTING);
    // We give IndexWriter a "huge" (256 MB) buffer, so it won't flush on its own unless the ES indexing buffer is also huge and/or
    // there are not too many shards allocated to this node.  Instead, IndexingMemoryController periodically checks
    // and refreshes the most heap-consuming shards when total indexing heap usage across all shards is too high:
    indexingBufferSize = new ByteSizeValue(256, ByteSizeUnit.MB);
    this.translogRecoveryPerformer = translogRecoveryPerformer;
    this.queryCache = queryCache;
    this.queryCachingPolicy = queryCachingPolicy;
    this.translogConfig = translogConfig;
    this.flushMergesAfter = flushMergesAfter;
    this.openMode = openMode;
    this.refreshListeners = refreshListeners;
    assert maxUnsafeAutoIdTimestamp >= IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP :
        "maxUnsafeAutoIdTimestamp must be >= -1 but was " + maxUnsafeAutoIdTimestamp;
    this.maxUnsafeAutoIdTimestamp = maxUnsafeAutoIdTimestamp;
}
 

import org.apache.lucene.search.similarities.Similarity; //导入依赖的package包/类
public void testAddSimilarity() throws IOException {
    Settings indexSettings = Settings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("index.similarity.my_similarity.type", "test_similarity")
            .put("index.similarity.my_similarity.key", "there is a key")
            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
            .build();
    IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
            new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
    module.addSimilarity("test_similarity", (string, providerSettings, indexLevelSettings) -> new SimilarityProvider() {
        @Override
        public String name() {
            return string;
        }

        @Override
        public Similarity get() {
            return new TestSimilarity(providerSettings.get("key"));
        }
    });

    IndexService indexService = newIndexService(module);
    SimilarityService similarityService = indexService.similarityService();
    assertNotNull(similarityService.getSimilarity("my_similarity"));
    assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity);
    assertEquals("my_similarity", similarityService.getSimilarity("my_similarity").name());
    assertEquals("there is a key", ((TestSimilarity) similarityService.getSimilarity("my_similarity").get()).key);
    indexService.close("simon says", false);
}