Java OpenMode.CREATE属性代码示例

static void indexDoc(IndexWriter writer, Path file, long lastModified) throws IOException {
	try (InputStream stream = Files.newInputStream(file)) {
		Document doc = new Document();
		Field pathField = new StringField("path", file.toString(), Field.Store.YES);
		doc.add(pathField);
		doc.add(new LongPoint("modified", lastModified));
		doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))));

		if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
			System.out.println("adding " + file);
			writer.addDocument(doc);
		} else {
			System.out.println("updating " + file);
			writer.updateDocument(new Term("path", file.toString()), doc);
		}
	}
}
 

/**
 * {@link PersistentSnapshotDeletionPolicy} wraps another
 * {@link IndexDeletionPolicy} to enable flexible snapshotting.
 * 
 * @param primary
 *          the {@link IndexDeletionPolicy} that is used on non-snapshotted
 *          commits. Snapshotted commits, by definition, are not deleted until
 *          explicitly released via {@link #release}.
 * @param dir
 *          the {@link Directory} which will be used to persist the snapshots
 *          information.
 * @param mode
 *          specifies whether a new index should be created, deleting all
 *          existing snapshots information (immediately), or open an existing
 *          index, initializing the class with the snapshots information.
 */
public PersistentSnapshotDeletionPolicy(IndexDeletionPolicy primary,
    Directory dir, OpenMode mode) throws IOException {
  super(primary);

  this.dir = dir;

  if (mode == OpenMode.CREATE) {
    clearPriorSnapshots();
  }

  loadPriorSnapshots();

  if (mode == OpenMode.APPEND && nextWriteGen == 0) {
    throw new IllegalStateException("no snapshots stored in this directory");
  }
}
 

/**
 * Indexes a single document
 * 
 * @throws TikaException
 * @throws SAXException
 */
public static void indexDoc(IndexWriter writer, Path file, TextArea results, long lastModified)
		throws IOException, SAXException, TikaException {
	AutoDetectParser parser = new AutoDetectParser();
	BodyContentHandler handler = new BodyContentHandler();
	Metadata metadata = new Metadata();
	try (InputStream stream = Files.newInputStream(file)) {
		parser.parse(stream, handler, metadata);
		Document doc = new Document();
		String[] metadataNames = metadata.names();
		for (String name : metadataNames)
			doc.add(new TextField(name, metadata.get(name), Field.Store.YES));
		doc.add(new StringField("path", file.toString(), Field.Store.YES));
		doc.add(new LongPoint("modified", lastModified));
		results.appendText("Title: " + metadata.get("title") + "\n");
		results.appendText("Artists: " + metadata.get("xmpDM:artist") + "\n");
		results.appendText("Genre: " + metadata.get("xmpDM:genre") + "\n");
		results.appendText("Year: " + metadata.get("xmpDM:releaseDate") + "\n");
		if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
			// New index, so we just add the document (no old document can
			// be there):
			results.appendText("adding " + file + "\n");
			writer.addDocument(doc);
		} else {
			// Existing index (an old copy of this document may have been
			// indexed):
			results.appendText("updating " + file);
			writer.updateDocument(new Term("path", file.toString()), doc);
		}
	}
}
 

static void indexDoc(IndexWriter writer, Path file, long lastModified) throws IOException {
    long counter = COUNTER.incrementAndGet();
    if (counter > MAX_ITEMS) {
        return;
    }
    System.out.println("Counter: " + counter);

    try (InputStream stream = Files.newInputStream(file)) {
        Document doc = new Document();

        Field pathField = new StringField("path", file.toString(), Field.Store.YES);
        doc.add(pathField);

        doc.add(new LongField("modified", lastModified, Field.Store.NO));

        doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(stream,
                StandardCharsets.UTF_8))));

        if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
            System.out.println("adding " + file);
            writer.addDocument(doc);
        } else {
            System.out.println("updating " + file);
            writer.updateDocument(new Term("path", file.toString()), doc);
        }
    } finally {
        long jobsDone = JOBS_DONE.incrementAndGet();
        System.out.println("Jobs done: " + jobsDone);
    }
}
 

@Test
public void testDefault() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
  FacetsConfig config = getConfig();

  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  // Obtain facets results and hand-test them
  assertCorrectResults(getTaxonomyFacetCounts(tr, config, sfc));

  assertOrdinalsExist("$facets", ir);

  IOUtils.close(tr, ir, iw, tw, indexDir, taxoDir);
}
 

@Test
public void testCustom() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Author", "$author");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  facetsMap.put("Author", getTaxonomyFacetCounts(tr, config, sfc, "$author"));
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);

  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$author", ir);

  IOUtils.close(tr, ir, iw, tw, indexDir, taxoDir);
}
 

@Test
public void testDifferentFieldsAndText() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();

  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Band", "$bands");
  config.setIndexFieldName("Composer", "$composers");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  facetsMap.put("Band", getTaxonomyFacetCounts(tr, config, sfc, "$bands"));
  facetsMap.put("Composer", getTaxonomyFacetCounts(tr, config, sfc, "$composers"));
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);
  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$bands", ir);
  assertOrdinalsExist("$composers", ir);

  IOUtils.close(tr, ir, iw, tw, indexDir, taxoDir);
}
 

@Test
public void testRecreateAndRefresh() throws Exception {
  // DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
  // CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
  // DirTaxoReader succeeding to refresh().
  Directory dir = newDirectory();
  
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
  touchTaxo(taxoWriter, new FacetLabel("a"));
  
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);

  touchTaxo(taxoWriter, new FacetLabel("b"));
  
  TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
  taxoReader.close();
  taxoReader = newtr;
  assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));

  // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
  taxoWriter.close();
  taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
  touchTaxo(taxoWriter, new FacetLabel("c"));
  taxoWriter.close();
  
  taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
  touchTaxo(taxoWriter, new FacetLabel("d"));
  taxoWriter.close();

  newtr = TaxonomyReader.openIfChanged(taxoReader);
  taxoReader.close();
  taxoReader = newtr;
  assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));

  taxoReader.close();
  dir.close();
}
 

@Test
public void testReaderFreshness() throws Exception {
  // ensures that the internal index reader is always kept fresh. Previously,
  // this simple scenario failed, if the cache just evicted the category that
  // is being added.
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
  int o1 = taxoWriter.addCategory(new FacetLabel("a"));
  int o2 = taxoWriter.addCategory(new FacetLabel("a"));
  assertTrue("ordinal for same category that is added twice should be the same !", o1 == o2);
  taxoWriter.close();
  dir.close();
}
 

@Test
public void testOpenIfChangedReuseAfterRecreate() throws Exception {
  // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
  FacetLabel cp_a = new FacetLabel("a");
  writer.addCategory(cp_a);
  writer.close();
  
  DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
  // fill r1's caches
  assertEquals(1, r1.getOrdinal(cp_a));
  assertEquals(cp_a, r1.getPath(1));
  
  // now recreate, add a different category
  writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
  FacetLabel cp_b = new FacetLabel("b");
  writer.addCategory(cp_b);
  writer.close();
  
  DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
  assertNotNull(r2);
  
  // fill r2's caches
  assertEquals(1, r2.getOrdinal(cp_b));
  assertEquals(cp_b, r2.getPath(1));
  
  // check that r1 doesn't see cp_b
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
  assertEquals(cp_a, r1.getPath(1));

  // check that r2 doesn't see cp_a
  assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
  assertEquals(cp_b, r2.getPath(1));

  r2.close();
  r1.close();
  dir.close();
}
 

@Test
public void testNoSnapshotInfos() throws Exception {
  Directory dir = newDirectory();
  new PersistentSnapshotDeletionPolicy(
      new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE);
  dir.close();
}
 

private void addDocument(final File file, final Document doc) throws IOException {
    if (indexWriter.getConfig().getOpenMode() == OpenMode.CREATE) {
        indexWriter.addDocument(doc);
    } else {
        Term term = termFactory.createTerm(fieldName, file.getPath());
        indexWriter.updateDocument(term, doc);
    }
}
 

static void indexDoc(final IndexWriter writer, final Path file, final long lastModified) {
    long counter = COUNTER.incrementAndGet();
    if (counter > MAX_ITEMS) {
        return;
    }
    System.out.println("Counter: " + counter);

    Runnable command = new Runnable() {
        @Override
        public void run() {
            try (InputStream stream = Files.newInputStream(file)) {
                Document doc = new Document();

                Field pathField = new StringField("path", file.toString(), Field.Store.YES);
                doc.add(pathField);

                doc.add(new LongField("modified", lastModified, Field.Store.NO));

                doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(
                        stream, StandardCharsets.UTF_8))));

                if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
                    System.out.println("adding " + file);
                    writer.addDocument(doc);
                } else {
                    System.out.println("updating " + file);
                    writer.updateDocument(new Term("path", file.toString()), doc);
                }
            } catch (IOException e) {
                e.printStackTrace();
                try {
                    writer.rollback();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            } finally {
                long jobsDone = JOBS_DONE.incrementAndGet();
                System.out.println("Jobs done: " + jobsDone);
            }
        }
    };
    ES.execute(command);
}
 

@Test
public void testTwoCustomsSameField() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Band", "$music");
  config.setIndexFieldName("Composer", "$music");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  Facets facets2 = getTaxonomyFacetCounts(tr, config, sfc, "$music");
  facetsMap.put("Band", facets2);
  facetsMap.put("Composer", facets2);
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);

  assertOrdinalsExist("$facets", ir);
  assertOrdinalsExist("$music", ir);
  assertOrdinalsExist("$music", ir);

  IOUtils.close(tr, ir, iw, tw, indexDir, taxoDir);
}
 

@Test
public void testSomeSameSomeDifferent() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();
  
  // create and open an index writer
  RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
      new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
  // create and open a taxonomy writer
  TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);

  FacetsConfig config = getConfig();
  config.setIndexFieldName("Band", "$music");
  config.setIndexFieldName("Composer", "$music");
  config.setIndexFieldName("Author", "$literature");
  seedIndex(tw, iw, config);

  IndexReader ir = iw.getReader();
  tw.commit();

  // prepare index reader and taxonomy.
  TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);

  // prepare searcher to search against
  IndexSearcher searcher = newSearcher(ir);

  FacetsCollector sfc = performSearch(tr, ir, searcher);

  Map<String,Facets> facetsMap = new HashMap<>();
  Facets facets2 = getTaxonomyFacetCounts(tr, config, sfc, "$music");
  facetsMap.put("Band", facets2);
  facetsMap.put("Composer", facets2);
  facetsMap.put("Author", getTaxonomyFacetCounts(tr, config, sfc, "$literature"));
  Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));

  // Obtain facets results and hand-test them
  assertCorrectResults(facets);
  assertOrdinalsExist("$music", ir);
  assertOrdinalsExist("$literature", ir);

  IOUtils.close(tr, ir, iw, tw);
  IOUtils.close(indexDir, taxoDir);
}