Java LogMergePolicy类代码示例

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
static MergePolicy newSortingMergePolicy(Sort sort) {
  // usually create a MP with a low merge factor so that many merges happen
  MergePolicy mp;
  int thingToDo = random().nextInt(3);
  if (thingToDo == 0) {
    TieredMergePolicy tmp = newTieredMergePolicy(random());
    final int numSegs = TestUtil.nextInt(random(), 3, 5);
    tmp.setSegmentsPerTier(numSegs);
    tmp.setMaxMergeAtOnce(TestUtil.nextInt(random(), 2, numSegs));
    mp = tmp;
  } else if (thingToDo == 1) {
    LogMergePolicy lmp = newLogMergePolicy(random());
    lmp.setMergeFactor(TestUtil.nextInt(random(), 3, 5));
    mp = lmp;
  } else {
    // just a regular random one from LTC (could be alcoholic etc)
    mp = newMergePolicy();
  }
  // wrap it with a sorting mp
  return new SortingMergePolicy(mp, sort);
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
/** just tries to configure things to keep the open file
 * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
  // keep number of open files lowish
  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    lmp.setNoCFSRatio(1.0);
  } else if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
    tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    tmp.setNoCFSRatio(1.0);
  }
  MergeScheduler ms = w.getConfig().getMergeScheduler();
  if (ms instanceof ConcurrentMergeScheduler) {
    // wtf... shouldnt it be even lower since its 1 by default?!?!
    ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
  }
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public void testSubclassConcurrentMergeScheduler() throws IOException {
  MockDirectoryWrapper dir = newMockDirectory();
  dir.failOn(new FailOnlyOnMerge());

  Document doc = new Document();
  Field idField = newStringField("id", "", Field.Store.YES);
  doc.add(idField);
  
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
      .setMergeScheduler(new MyMergeScheduler())
      .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setMergePolicy(newLogMergePolicy()));
  LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy();
  logMP.setMergeFactor(10);
  for(int i=0;i<20;i++)
    writer.addDocument(doc);

  ((MyMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
  writer.close();
  
  assertTrue(mergeThreadCreated);
  assertTrue(mergeCalled);
  assertTrue(excCalled);
  dir.close();
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
/** just tries to configure things to keep the open file
 * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
  // keep number of open files lowish
  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    lmp.setUseCompoundFile(true);
  } else if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
    tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    tmp.setUseCompoundFile(true);
  }
  MergeScheduler ms = w.getConfig().getMergeScheduler();
  if (ms instanceof ConcurrentMergeScheduler) {
    ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
    ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
  }
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public void testSubclassConcurrentMergeScheduler() throws IOException {
  MockDirectoryWrapper dir = newMockDirectory();
  dir.failOn(new FailOnlyOnMerge());

  Document doc = new Document();
  Field idField = newStringField("id", "", Field.Store.YES);
  doc.add(idField);
  
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
      TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergeScheduler(new MyMergeScheduler())
      .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setMergePolicy(newLogMergePolicy()));
  LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy();
  logMP.setMergeFactor(10);
  for(int i=0;i<20;i++)
    writer.addDocument(doc);

  ((MyMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
  writer.close();
  
  assertTrue(mergeThreadCreated);
  assertTrue(mergeCalled);
  assertTrue(excCalled);
  dir.close();
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
static MergePolicy newSortingMergePolicy(Sorter sorter) {
  // create a MP with a low merge factor so that many merges happen
  MergePolicy mp;
  if (random().nextBoolean()) {
    TieredMergePolicy tmp = newTieredMergePolicy(random());
    final int numSegs = _TestUtil.nextInt(random(), 3, 5);
    tmp.setSegmentsPerTier(numSegs);
    tmp.setMaxMergeAtOnce(_TestUtil.nextInt(random(), 2, numSegs));
    mp = tmp;
  } else {
    LogMergePolicy lmp = newLogMergePolicy(random());
    lmp.setMergeFactor(_TestUtil.nextInt(random(), 3, 5));
    mp = lmp;
  }
  // wrap it with a sorting mp
  return new SortingMergePolicy(mp, sorter);
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public static IndexWriter getIndexWriter(String indexPath, boolean create) throws IOException {
    Directory dir = FSDirectory.open(Paths.get(indexPath));
    Analyzer analyzer = new SmartChineseAnalyzer();
    IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
    LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
    mergePolicy.setMergeFactor(50);
    mergePolicy.setMaxMergeDocs(5000);
    if (create){
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    } else {
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    }
    return new IndexWriter(dir, iwc);
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public static LogMergePolicy newLogMergePolicy(Random r) {
  LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
  logmp.setCalibrateSizeByDeletes(r.nextBoolean());
  if (rarely(r)) {
    logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9));
  } else {
    logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50));
  }
  configureRandom(r, logmp);
  return logmp;
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public void testLogMergePolicyConfig() throws Exception {
  
  final Class<? extends LogMergePolicy> mpClass = random().nextBoolean()
    ? LogByteSizeMergePolicy.class : LogDocMergePolicy.class;

  System.setProperty("solr.test.log.merge.policy", mpClass.getName());

  initCore("solrconfig-logmergepolicy.xml","schema-minimal.xml");
  IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());

  // verify some props set to -1 get lucene internal defaults
  assertEquals(-1, solrConfig.indexConfig.maxBufferedDocs);
  assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, 
               iwc.getMaxBufferedDocs());
  assertEquals(-1, solrConfig.indexConfig.maxIndexingThreads);
  assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, 
               iwc.getMaxThreadStates());
  assertEquals(-1, solrConfig.indexConfig.ramBufferSizeMB, 0.0D);
  assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, 
               iwc.getRAMBufferSizeMB(), 0.0D);


  LogMergePolicy logMP = assertAndCast(mpClass, iwc.getMergePolicy());

  // set by legacy <mergeFactor> setting
  assertEquals(11, logMP.getMergeFactor());
  // set by legacy <maxMergeDocs> setting
  assertEquals(456, logMP.getMaxMergeDocs());

}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public static void setUseCompoundFile(MergePolicy mp, boolean v) {
  if (mp instanceof TieredMergePolicy) {
    ((TieredMergePolicy) mp).setUseCompoundFile(v);
  } else if (mp instanceof LogMergePolicy) {
    ((LogMergePolicy) mp).setUseCompoundFile(v);
  } else {
    throw new IllegalArgumentException("cannot set compound file for MergePolicy " + mp);
  }
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
@Override
public synchronized boolean clearWorkflowInstances() throws InstanceRepositoryException {
  IndexWriter writer = null;
  try {
      IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
      config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
      LogMergePolicy lmp =new LogDocMergePolicy();
      lmp.setMergeFactor(mergeFactor);
      config.setMergePolicy(lmp);

      writer = new IndexWriter(indexDir, config);
      LOG.log(Level.FINE,
              "LuceneWorkflowEngine: remove all workflow instances");
      writer.deleteDocuments(new Term("myfield", "myvalue"));
  } catch (IOException e) {
      LOG.log(Level.SEVERE, e.getMessage());
      LOG
              .log(Level.WARNING,
                      "Exception removing workflow instances from index: Message: "
                              + e.getMessage());
      throw new InstanceRepositoryException(e.getMessage());
  } finally {
    if (writer != null){
      try{
        writer.close();
      }
      catch(Exception ignore){}
      
      writer = null;
    }

  }
  
  return true;
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
public JsonObject asJson() {
    JsonArrayBuilder strategiesJsonBuilder = Json.createArrayBuilder();
    for (ClusterStrategy strategy : this.clusterConfig.strategies) {
        strategiesJsonBuilder.add(Json.createObjectBuilder()
                .add("clusteringEps", strategy.clusteringEps)
                .add("clusteringMinPoints", strategy.clusteringMinPoints));
    }

    JsonObject json = Json.createObjectBuilder()
            .add("similarity", similarity.toString())
            .add("mergePolicy", this.mergePolicy instanceof TieredMergePolicy
                    ? Json.createObjectBuilder()
                            .add("type", "TieredMergePolicy")
                            .add("maxMergeAtOnce", ((TieredMergePolicy) this.mergePolicy).getMaxMergeAtOnce())
                            .add("segmentsPerTier", ((TieredMergePolicy) this.mergePolicy).getSegmentsPerTier())
                    : Json.createObjectBuilder()
                            .add("type", "LogDocMergePolicy")
                            .add("maxMergeDocs", ((LogMergePolicy) this.mergePolicy).getMaxMergeDocs())
                            .add("mergeFactor", ((LogMergePolicy) this.mergePolicy).getMergeFactor()))
            .add("lruTaxonomyWriterCacheSize", lruTaxonomyWriterCacheSize)
            .add("numberOfConcurrentTasks", numberOfConcurrentTasks)
            .add("commitCount", commitCount)
            .add("commitTimeout", commitTimeout)
            .add("cacheFacetOrdinals", this.cacheFacetOrdinals)
            .add("clustering", Json.createObjectBuilder()
                    .add("clusterMoreRecords", clusterConfig.clusterMoreRecords)
                    .add("strategies", strategiesJsonBuilder))
            .build();
    return json;
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
/**
 * Test that IndexWriter settings stick.
 */
public void testIndexWriterSettings() throws Exception {
  // 1. alg definition (required in every "logic" test)
  String algLines[] = {
      "# ----- properties ",
      "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
      "docs.file=" + getReuters20LinesFile(),
      "content.source.log.step=3",
      "ram.flush.mb=-1",
      "max.buffered=2",
      "compound=cmpnd:true:false",
      "doc.term.vector=vector:false:true",
      "content.source.forever=false",
      "directory=RAMDirectory",
      "doc.stored=false",
      "merge.factor=3",
      "doc.tokenized=false",
      "debug.level=1",
      "# ----- alg ",
      "{ \"Rounds\"",
      "  ResetSystemErase",
      "  CreateIndex",
      "  { \"AddDocs\"  AddDoc > : * ",
      "  NewRound",
      "} : 2",
  };

  // 2. execute the algorithm  (required in every "logic" test)
  Benchmark benchmark = execBenchmark(algLines);
  final IndexWriter writer = benchmark.getRunData().getIndexWriter();
  assertEquals(2, writer.getConfig().getMaxBufferedDocs());
  assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, (int) writer.getConfig().getRAMBufferSizeMB());
  assertEquals(3, ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor());
  assertEquals(0.0d, writer.getConfig().getMergePolicy().getNoCFSRatio(), 0.0);
  writer.close();
  Directory dir = benchmark.getRunData().getDirectory();
  IndexReader reader = DirectoryReader.open(dir);
  Fields tfv = reader.getTermVectors(0);
  assertNotNull(tfv);
  assertTrue(tfv.size() > 0);
  reader.close();
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
@Test
public void testOpenIfChangedManySegments() throws Exception {
  // test openIfChanged() when the taxonomy contains many segments
  Directory dir = newDirectory();
  
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
    @Override
    protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
      IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
      lmp.setMergeFactor(2);
      return conf;
    }
  };
  TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
  
  int numRounds = random().nextInt(10) + 10;
  int numCategories = 1; // one for root
  for (int i = 0; i < numRounds; i++) {
    int numCats = random().nextInt(4) + 1;
    for (int j = 0; j < numCats; j++) {
      writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j)));
    }
    numCategories += numCats + 1 /* one for round-parent */;
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
    assertNotNull(newtr);
    reader.close();
    reader = newtr;
    
    // assert categories
    assertEquals(numCategories, reader.getSize());
    int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i)));
    int[] parents = reader.getParallelTaxonomyArrays().parents();
    assertEquals(0, parents[roundOrdinal]); // round's parent is root
    for (int j = 0; j < numCats; j++) {
      int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j)));
      assertEquals(roundOrdinal, parents[ord]); // round's parent is root
    }
  }

  reader.close();
  writer.close();
  dir.close();
}
 

import org.apache.lucene.index.LogMergePolicy; //导入依赖的package包/类
/** we will manually instantiate preflex-rw here */
@BeforeClass
public static void beforeClass() throws Exception {
  // NOTE: turn off compound file, this test will open some index files directly.
  LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
  IndexWriterConfig config = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))
                               .setUseCompoundFile(false);
  
  termIndexInterval = config.getTermIndexInterval();
  indexDivisor = TestUtil.nextInt(random(), 1, 10);
  NUMBER_OF_DOCUMENTS = atLeast(100);
  NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS));
  
  directory = newDirectory();

  config.setCodec(new PreFlexRWCodec());
  LogMergePolicy mp = newLogMergePolicy();
  // NOTE: turn off compound file, this test will open some index files directly.
  mp.setNoCFSRatio(0.0);
  config.setMergePolicy(mp);

  
  populate(directory, config);

  DirectoryReader r0 = IndexReader.open(directory);
  SegmentReader r = LuceneTestCase.getOnlySegmentReader(r0);
  String segment = r.getSegmentName();
  r.close();

  FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader();
  FieldInfos fieldInfos = infosReader.read(directory, segment, "", IOContext.READONCE);
  String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION);
  long tiiFileLength = directory.fileLength(segmentFileName);
  IndexInput input = directory.openInput(segmentFileName, newIOContext(random()));
  termEnum = new PreflexRWSegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random())), fieldInfos, false);
  int totalIndexInterval = termEnum.indexInterval * indexDivisor;
  
  SegmentTermEnum indexEnum = new PreflexRWSegmentTermEnum(input, fieldInfos, true);
  index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval);
  indexEnum.close();
  input.close();
  
  reader = IndexReader.open(directory);
  sampleTerms = sample(random(),reader,1000);
}