Java IndexWriterConfig.setIndexDeletionPolicy方法代码示例

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
private IndexWriterConfig getIndexWriterConfig(boolean create) {
    final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
    iwc.setCommitOnClose(false); // we by default don't commit on close
    iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND);
    iwc.setIndexDeletionPolicy(deletionPolicy);
    // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
    boolean verbose = false;
    try {
        verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
    } catch (Exception ignore) {
    }
    iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
    iwc.setMergeScheduler(mergeScheduler);
    MergePolicy mergePolicy = config().getMergePolicy();
    // Give us the opportunity to upgrade old segments while performing
    // background merges
    mergePolicy = new ElasticsearchMergePolicy(mergePolicy);
    iwc.setMergePolicy(mergePolicy);
    iwc.setSimilarity(engineConfig.getSimilarity());
    iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
    iwc.setCodec(engineConfig.getCodec());
    iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
    return iwc;
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testNoCommit() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
    fail("should have failed when there are no commits to snapshot");
  } catch (IllegalStateException e) {
    // expected
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testSegmentsFileLast() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    assertEquals(2, sourceFiles.size());
    for (List<RevisionFile> files : sourceFiles.values()) {
      String lastFile = files.get(files.size() - 1).fileName;
      assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS) && !lastFile.equals(IndexFileNames.SEGMENTS_GEN));
    }
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  publishDir = newMockDirectory();
  handlerDir = newMockDirectory();
  sourceDirFactory = new PerSessionDirectoryFactory(createTempDir("replicationClientTest"));
  replicator = new LocalReplicator();
  callback = new IndexReadyCallback(handlerDir);
  handler = new IndexReplicationHandler(handlerDir, callback);
  client = new ReplicationClient(replicator, handler, sourceDirFactory);
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  publishWriter = new IndexWriter(publishDir, conf);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testSegmentsFileLast() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    assertEquals(1, sourceFiles.size());
    List<RevisionFile> files = sourceFiles.values().iterator().next();
    String lastFile = files.get(files.size() - 1).fileName;
    assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS) && !lastFile.equals(IndexFileNames.SEGMENTS_GEN));
  } finally {
    IOUtils.close(writer, dir);
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Before
@Override
public void setUp() throws Exception {
  super.setUp();
  if (VERBOSE) {
    System.setProperty("org.eclipse.jetty.LEVEL", "DEBUG"); // sets stderr logging to DEBUG level
  }
  clientWorkDir = createTempDir("httpReplicatorTest");
  handlerIndexDir = newDirectory();
  serverIndexDir = newDirectory();
  serverReplicator = new LocalReplicator();
  startServer();
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  writer = new IndexWriter(serverIndexDir, conf);
  reader = DirectoryReader.open(writer, false);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testSerialization() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  writer.addDocument(new Document());
  writer.commit();
  Revision rev = new IndexRevision(writer);
  
  SessionToken session1 = new SessionToken("17", rev);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  session1.serialize(new DataOutputStream(baos));
  byte[] b = baos.toByteArray();
  SessionToken session2 = new SessionToken(new DataInputStream(new ByteArrayInputStream(b)));
  assertEquals(session1.id, session2.id);
  assertEquals(session1.version, session2.version);
  assertEquals(1, session2.sourceFiles.size());
  assertEquals(session1.sourceFiles.size(), session2.sourceFiles.size());
  assertEquals(session1.sourceFiles.keySet(), session2.sourceFiles.keySet());
  List<RevisionFile> files1 = session1.sourceFiles.values().iterator().next();
  List<RevisionFile> files2 = session2.sourceFiles.values().iterator().next();
  assertEquals(files1, files2);
  
  IOUtils.close(writer, dir);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  publishIndexDir = newDirectory();
  publishTaxoDir = newDirectory();
  handlerIndexDir = newMockDirectory();
  handlerTaxoDir = newMockDirectory();
  clientWorkDir = createTempDir("replicationClientTest");
  sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
  replicator = new LocalReplicator();
  callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
  handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
  client = new ReplicationClient(replicator, handler, sourceDirFactory);
  
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  publishIndexWriter = new IndexWriter(publishIndexDir, conf);
  publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
  config = new FacetsConfig();
  config.setHierarchical("A", true);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
private static IndexWriterConfig standardConfig()
{
    IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER );

    writerConfig.setMaxBufferedDocs( 100000 ); // TODO figure out depending on environment?
    writerConfig.setIndexDeletionPolicy( new MultipleBackupDeletionPolicy() );
    writerConfig.setUseCompoundFile( true );

    // TODO: TieredMergePolicy & possibly SortingMergePolicy
    LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy();
    mergePolicy.setNoCFSRatio( 1.0 );
    mergePolicy.setMinMergeMB( 0.1 );
    mergePolicy.setMergeFactor( 2 );
    writerConfig.setMergePolicy( mergePolicy );

    return writerConfig;
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
public void testUserDataRead() throws IOException {
    final ShardId shardId = new ShardId("index", "_na_", 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId));
    IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec());
    SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    config.setIndexDeletionPolicy(deletionPolicy);
    IndexWriter writer = new IndexWriter(store.directory(), config);
    Document doc = new Document();
    doc.add(new TextField("id", "1", Field.Store.NO));
    writer.addDocument(doc);
    Map<String, String> commitData = new HashMap<>(2);
    String syncId = "a sync id";
    String translogId = "a translog id";
    commitData.put(Engine.SYNC_COMMIT_ID, syncId);
    commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogId);
    writer.setCommitData(commitData);
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata;
    metadata = store.getMetadata(randomBoolean() ? null : deletionPolicy.snapshot());
    assertFalse(metadata.asMap().isEmpty());
    // do not check for correct files, we have enough tests for that above
    assertThat(metadata.getCommitUserData().get(Engine.SYNC_COMMIT_ID), equalTo(syncId));
    assertThat(metadata.getCommitUserData().get(Translog.TRANSLOG_GENERATION_KEY), equalTo(translogId));
    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
/**
 * Ritorna l'indexwriter corretto per accedere all'indice.
 *
 * @param indexDir cartella indice
 * @param optimize true se si vuole ottimizzare
 * @param openMode indice aperto in append o in create
 * @param analyzer analyzer da utilizzare
 * @return IndexWriter
 * @throws Exception Eccezione eccezione
 */
public static IndexWriter getIndexWriter(Path indexDir, boolean optimize, IndexWriterConfig.OpenMode openMode, Analyzer analyzer) throws Exception {
    Directory fsDir = FSDirectory.open(indexDir);
    IndexWriterConfig iwConf = new IndexWriterConfig(analyzer);
    iwConf.setOpenMode(openMode);
    if (optimize) {
        iwConf.setIndexDeletionPolicy(new KeepLastIndexDeletionPolicy());
        iwConf.setUseCompoundFile(true);
    }
    return new IndexWriter(fsDir, iwConf);
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
  IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
  sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
  conf.setIndexDeletionPolicy(sdp);
  return conf;
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Test
public void testRevisionRelease() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  // we look to see that certain files are deleted:
  if (indexDir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)indexDir).setEnableVirusScanner(false);
  }
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    // releasing that revision should not delete the files
    rev1.release();
    assertTrue(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
    assertTrue(slowFileExists(taxoDir, IndexFileNames.SEGMENTS + "_1"));
    
    rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); // create revision again, so the files are snapshotted
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
    rev1.release(); // this release should trigger the delete of segments_1
    assertFalse(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
    if (indexDir instanceof MockDirectoryWrapper) {
      // set back to on for other tests
      ((MockDirectoryWrapper)indexDir).setEnableVirusScanner(true);
    }
  }
}
 

import org.apache.lucene.index.IndexWriterConfig; //导入方法依赖的package包/类
@Before
@Override
public void setUp() throws Exception {
  super.setUp();
  sourceDir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  sourceWriter = new IndexWriter(sourceDir, conf);
  replicator = new LocalReplicator();
}