Java IndexWriterConfig.setMaxBufferedDocs方法代码示例

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
private static IndexWriterConfig standardConfig()
{
    IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER );

    writerConfig.setMaxBufferedDocs( 100000 ); // TODO figure out depending on environment?
    writerConfig.setIndexDeletionPolicy( new MultipleBackupDeletionPolicy() );
    writerConfig.setUseCompoundFile( true );

    // TODO: TieredMergePolicy & possibly SortingMergePolicy
    LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy();
    mergePolicy.setNoCFSRatio( 1.0 );
    mergePolicy.setMinMergeMB( 0.1 );
    mergePolicy.setMergeFactor( 2 );
    writerConfig.setMergePolicy( mergePolicy );

    return writerConfig;
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/**
 * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search.
 */
public void testSampler() throws IOException {
    TextFieldType textFieldType = new TextFieldType();
    textFieldType.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.GLOBAL, new StandardAnalyzer()));
    MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    numericFieldType.setName("int");

    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
    indexWriterConfig.setMaxBufferedDocs(100);
    indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment with predictable docIds
    try (Directory dir = newDirectory();
            IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
        for (long value : new long[] {7, 3, -10, -6, 5, 50}) {
            Document doc = new Document();
            StringBuilder text = new StringBuilder();
            for (int i = 0; i < value; i++) {
                text.append("good ");
            }
            doc.add(new Field("text", text.toString(), textFieldType));
            doc.add(new SortedNumericDocValuesField("int", value));
            w.addDocument(doc);
        }

        SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler")
                .shardSize(3)
                .subAggregation(new MinAggregationBuilder("min")
                        .field("int"));
        try (IndexReader reader = DirectoryReader.open(w)) {
            assertEquals("test expects a single segment", 1, reader.leaves().size());
            IndexSearcher searcher = new IndexSearcher(reader);
            Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "good")), aggBuilder, textFieldType,
                    numericFieldType);
            Min min = sampler.getAggregations().get("min");
            assertEquals(5.0, min.getValue(), 0);
        }
    }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/** Test that core cache key (needed for NRT) is working */
public void testCoreCacheKey() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(null);
    iwc.setMaxBufferedDocs(100);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    IndexWriter iw = new IndexWriter(dir, iwc);
    
    // add two docs, id:0 and id:1
    Document doc = new Document();
    Field idField = new StringField("id", "", Field.Store.NO);
    doc.add(idField);
    idField.setStringValue("0");
    iw.addDocument(doc);
    idField.setStringValue("1");
    iw.addDocument(doc);
    
    // open reader
    ShardId shardId = new ShardId("fake", "_na_", 1);
    DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(iw), shardId);
    assertEquals(2, ir.numDocs());
    assertEquals(1, ir.leaves().size());

    // delete id:0 and reopen
    iw.deleteDocuments(new Term("id", "0"));
    DirectoryReader ir2 = DirectoryReader.openIfChanged(ir);
    
    // we should have the same cache key as before
    assertEquals(1, ir2.numDocs());
    assertEquals(1, ir2.leaves().size());
    assertSame(ir.leaves().get(0).reader().getCoreCacheKey(), ir2.leaves().get(0).reader().getCoreCacheKey());
    IOUtils.close(ir, ir2, iw, dir);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/**
 * Opens all the needed streams that the engine needs to work properly.
 * 
 * @throws IndexException
 */
private void openStreams() throws IndexException {
	try {
		if (_nativeLocking) {
			_storage = FSDirectory.open(new File(INDEX_DIR), new NativeFSLockFactory(INDEX_DIR));
		} else {
			_storage = FSDirectory.open(new File(INDEX_DIR));
		}

		IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32 ,ANALYZER);
		conf.setMaxBufferedDocs(_maxDocsBuffer);
		conf.setRAMBufferSizeMB(_maxRAMBufferSize);

		_iWriter = new IndexWriter(_storage, conf);
	} catch (IOException e) {
		closeAll();

		throw new IndexException("Unable to initialize the index", e);
	}
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
protected static void createIndex(boolean doMultiSegment) throws Exception {
  if (VERBOSE) {
    System.out.println("TEST: setUp");
  }
  // prepare a small index with just a few documents.
  dir = newDirectory();
  anlzr = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(anlzr).setMergePolicy(newLogMergePolicy());
  if (doMultiSegment) {
    iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 7));
  }
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  // add docs not exactly in natural ID order, to verify we do check the order of docs by scores
  int remaining = N_DOCS;
  boolean done[] = new boolean[N_DOCS];
  int i = 0;
  while (remaining > 0) {
    if (done[i]) {
      throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!");
    }
    addDoc(iw, i);
    done[i] = true;
    i = (i + 4) % N_DOCS;
    remaining --;
  }
  if (!doMultiSegment) {
    if (VERBOSE) {
      System.out.println("TEST: setUp full merge");
    }
    iw.forceMerge(1);
  }
  iw.close();
  if (VERBOSE) {
    System.out.println("TEST: setUp done close");
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/** create a new index writer config with random defaults using the specified random */
public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) {
  IndexWriterConfig c = new IndexWriterConfig(v, a);
  c.setSimilarity(classEnvRule.similarity);
  if (VERBOSE) {
    // Even though TestRuleSetupAndRestoreClassEnv calls
    // InfoStream.setDefault, we do it again here so that
    // the PrintStreamInfoStream.messageID increments so
    // that when there are separate instances of
    // IndexWriter created we see "IW 0", "IW 1", "IW 2",
    // ... instead of just always "IW 0":
    c.setInfoStream(new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(System.out));
  }

  if (r.nextBoolean()) {
    c.setMergeScheduler(new SerialMergeScheduler());
  } else if (rarely(r)) {
    int maxThreadCount = TestUtil.nextInt(r, 1, 4);
    int maxMergeCount = TestUtil.nextInt(r, maxThreadCount, maxThreadCount+4);
    ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
    cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
    c.setMergeScheduler(cms);
  }
  if (r.nextBoolean()) {
    if (rarely(r)) {
      // crazy value
      c.setMaxBufferedDocs(TestUtil.nextInt(r, 2, 15));
    } else {
      // reasonable value
      c.setMaxBufferedDocs(TestUtil.nextInt(r, 16, 1000));
    }
  }
  if (r.nextBoolean()) {
    if (rarely(r)) {
      // crazy value
      c.setTermIndexInterval(r.nextBoolean() ? TestUtil.nextInt(r, 1, 31) : TestUtil.nextInt(r, 129, 1000));
    } else {
      // reasonable value
      c.setTermIndexInterval(TestUtil.nextInt(r, 32, 128));
    }
  }
  if (r.nextBoolean()) {
    int maxNumThreadStates = rarely(r) ? TestUtil.nextInt(r, 5, 20) // crazy value
        : TestUtil.nextInt(r, 1, 4); // reasonable value

    c.setMaxThreadStates(maxNumThreadStates);
  }

  c.setMergePolicy(newMergePolicy(r));

  if (rarely(r)) {
    c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream()));
  }
  c.setUseCompoundFile(r.nextBoolean());
  c.setReaderPooling(r.nextBoolean());
  c.setReaderTermsIndexDivisor(TestUtil.nextInt(r, 1, 4));
  c.setCheckIntegrityAtMerge(r.nextBoolean());
  return c;
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test(expected=IllegalArgumentException.class)
public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
  Directory dir = newDirectory();
  // test explicitly needs files to always be actually deleted
  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
  }
  IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
  iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
  iwConf.setCodec(CompressingCodec.randomInstance(random()));
  // disable CFS because this test checks file names
  iwConf.setMergePolicy(newLogMergePolicy(false));
  iwConf.setUseCompoundFile(false);

  // Cannot use RIW because this test wants CFS to stay off:
  IndexWriter iw = new IndexWriter(dir, iwConf);

  final Document validDoc = new Document();
  validDoc.add(new IntField("id", 0, Store.YES));
  iw.addDocument(validDoc);
  iw.commit();
  
  // make sure that #writeField will fail to trigger an abort
  final Document invalidDoc = new Document();
  FieldType fieldType = new FieldType();
  fieldType.setStored(true);
  invalidDoc.add(new Field("invalid", fieldType) {
    
    @Override
    public String stringValue() {
      // TODO: really bad & scary that this causes IW to
      // abort the segment!!  We should fix this.
      return null;
    }
    
  });
  
  try {
    iw.addDocument(invalidDoc);
    iw.commit();
  }
  finally {
    int counter = 0;
    for (String fileName : dir.listAll()) {
      if (fileName.endsWith(".fdt") || fileName.endsWith(".fdx")) {
        counter++;
      }
    }
    // Only one .fdt and one .fdx files must have been found
    assertEquals(2, counter);
    iw.close();
    dir.close();
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testChangeCodecAndMerge() throws IOException {
  Directory dir = newDirectory();
  if (VERBOSE) {
    System.out.println("TEST: make new index");
  }
  IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setOpenMode(OpenMode.CREATE).setCodec(new MockCodec());
  iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
  //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10);
  IndexWriter writer = newWriter(dir, iwconf);

  addDocs(writer, 10);
  writer.commit();
  assertQuery(new Term("content", "aaa"), dir, 10);
  if (VERBOSE) {
    System.out.println("TEST: addDocs3");
  }
  addDocs3(writer, 10);
  writer.commit();
  writer.close();

  assertQuery(new Term("content", "ccc"), dir, 10);
  assertQuery(new Term("content", "aaa"), dir, 10);
  Codec codec = iwconf.getCodec();

  iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
      .setOpenMode(OpenMode.APPEND).setCodec(codec);
  //((LogMergePolicy) iwconf.getMergePolicy()).setNoCFSRatio(0.0);
  //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10);
  iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);

  iwconf.setCodec(new MockCodec2()); // uses standard for field content
  writer = newWriter(dir, iwconf);
  // swap in new codec for currently written segments
  if (VERBOSE) {
    System.out.println("TEST: add docs w/ Standard codec for content field");
  }
  addDocs2(writer, 10);
  writer.commit();
  codec = iwconf.getCodec();
  assertEquals(30, writer.maxDoc());
  assertQuery(new Term("content", "bbb"), dir, 10);
  assertQuery(new Term("content", "ccc"), dir, 10);   ////
  assertQuery(new Term("content", "aaa"), dir, 10);

  if (VERBOSE) {
    System.out.println("TEST: add more docs w/ new codec");
  }
  addDocs2(writer, 10);
  writer.commit();
  assertQuery(new Term("content", "ccc"), dir, 10);
  assertQuery(new Term("content", "bbb"), dir, 20);
  assertQuery(new Term("content", "aaa"), dir, 10);
  assertEquals(40, writer.maxDoc());

  if (VERBOSE) {
    System.out.println("TEST: now optimize");
  }
  writer.forceMerge(1);
  assertEquals(40, writer.maxDoc());
  writer.close();
  assertQuery(new Term("content", "ccc"), dir, 10);
  assertQuery(new Term("content", "bbb"), dir, 20);
  assertQuery(new Term("content", "aaa"), dir, 10);

  dir.close();
}