Java IndexWriter.addDocument方法代码示例

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
private void addAppsToIndexerWriter(List<App> list, IndexWriter indexerWriter) {
    Field name = new Field(fieldName, "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
    NumericField catalog = new NumericField("catalog", Field.Store.NO, true);
    NumericField downloadRank = new NumericField("downloadRank", Field.Store.NO, true);
    for (App a : list) {
        try {
            Document doc = new Document();
            name.setValue(a.getName().toLowerCase());
            doc.add(name);
            downloadRank.setIntValue(a.getDownloadRank());
            doc.add(downloadRank);
            catalog.setIntValue(a.getCatalog());
            doc.add(catalog);
            indexerWriter.addDocument(doc);
        } catch (Exception e) {
            logger.error("Exception", e);
        }
    }
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
/** 
 * test version lookup actually works
 */
public void testSimple() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
    writer.addDocument(doc);
    DirectoryReader reader = DirectoryReader.open(writer);
    LeafReaderContext segment = reader.leaves().get(0);
    PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
    // found doc
    DocIdAndVersion result = lookup.lookup(new BytesRef("6"), null, segment);
    assertNotNull(result);
    assertEquals(87, result.version);
    assertEquals(0, result.docId);
    // not found doc
    assertNull(lookup.lookup(new BytesRef("7"), null, segment));
    // deleted doc
    assertNull(lookup.lookup(new BytesRef("6"), new Bits.MatchNoBits(1), segment));
    reader.close();
    writer.close();
    dir.close();
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void addIndex(UUser user) throws Exception {
  IndexWriter writer = getWriter();
  Document doc = new Document();
  /*
 * yes是会将数据存进索引，如果查询结果中需要将记录显示出来就要存进去，如果查询结果
 * 只是显示标题之类的就可以不用存，而且内容过长不建议存进去
 * 使用TextField类是可以用于查询的。
 */
  try {
    doc.add(new StringField("userid", String.valueOf(user.getId()), Field.Store.YES));
    doc.add(new TextField("username", user.getUsername(), Field.Store.YES));

    writer.addDocument(doc);
  } catch (Exception e) {
    e.printStackTrace();
    throw e;
  } finally {
    writer.close();
  }
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void doTestDocValueRangeQueries(NumberType type, Supplier<Number> valueSupplier) throws Exception {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
    final int numDocs = TestUtil.nextInt(random(), 100, 500);
    for (int i = 0; i < numDocs; ++i) {
        w.addDocument(type.createFields("foo", valueSupplier.get(), true, true, false));
    }
    DirectoryReader reader = DirectoryReader.open(w);
    IndexSearcher searcher = newSearcher(reader);
    w.close();
    final int iters = 10;
    for (int iter = 0; iter < iters; ++iter) {
        Query query = type.rangeQuery("foo",
                random().nextBoolean() ? null : valueSupplier.get(),
                random().nextBoolean() ? null : valueSupplier.get(),
                randomBoolean(), randomBoolean(), true);
        assertThat(query, Matchers.instanceOf(IndexOrDocValuesQuery.class));
        IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query;
        assertEquals(
                searcher.count(indexOrDvQuery.getIndexQuery()),
                searcher.count(indexOrDvQuery.getRandomAccessQuery()));
    }
    reader.close();
    dir.close();
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void testNRTSearchOnClosedWriter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    DirectoryReader reader = DirectoryReader.open(indexWriter);

    for (int i = 0; i < 100; i++) {
        Document document = new Document();
        TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES);
        field.setBoost(i);
        document.add(field);
        indexWriter.addDocument(document);
    }
    reader = refreshReader(reader);

    indexWriter.close();

    TermsEnum termDocs = SlowCompositeReaderWrapper.wrap(reader).terms("_id").iterator();
    termDocs.next();
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
/** Test that version map cache behaves properly with a filtered reader */
public void testCacheFilterReader() throws Exception {
    int size = Versions.lookupStates.size();

    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
    writer.addDocument(doc);
    DirectoryReader reader = DirectoryReader.open(writer);
    assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
    assertEquals(size+1, Versions.lookupStates.size());
    // now wrap the reader
    DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5));
    assertEquals(87, Versions.loadVersion(wrapped, new Term(UidFieldMapper.NAME, "6")));
    // same size map: core cache key is shared
    assertEquals(size+1, Versions.lookupStates.size());

    reader.close();
    writer.close();
    // core should be evicted from the map
    assertEquals(size, Versions.lookupStates.size());
    dir.close();
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void testSingleValued() throws IOException {
    Directory dir = newDirectory();
    // we need the default codec to check for singletons
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
        doc.add(f);
    }
    w.addDocument(doc);
    final DirectoryReader dirReader = DirectoryReader.open(w);
    LeafReader reader = getOnlyLeafReader(dirReader);
    SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData(
            reader, "half_float").getDoubleValues();
    assertNotNull(FieldData.unwrapSingleton(values));
    values.setDocument(0);
    assertEquals(1, values.count());
    assertEquals(3f, values.valueAt(0), 0f);
    IOUtils.close(dirReader, w, dir);
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
@Override
public <S, T> void store(Collection<T> toAdd, Collection<S> toDelete, Convertor<? super T, ? extends Document> docConvertor, Convertor<? super S, ? extends Query> queryConvertor, boolean optimize) throws IOException {
    lock.writeLock().lock();
    try {
        final IndexWriter out = getWriter();
        try {
            for (S td : toDelete) {
                out.deleteDocuments(queryConvertor.convert(td));
            }
            if (toAdd.isEmpty()) {
                return;
            }
            for (Iterator<T> it = toAdd.iterator(); it.hasNext();) {
                T entry = it.next();
                it.remove();
                final Document doc = docConvertor.convert(entry);
                out.addDocument(doc);
            }
        } finally {

            try {
                out.close();
            } finally {
                refreshReader();
            }
        }
    } finally {
        lock.writeLock().unlock();
    }
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void testRangeQuery() throws IOException {
    // make sure the accuracy loss of scaled floats only occurs at index time
    // this test checks that searching scaled floats yields the same results as
    // searching doubles that are rounded to the closest half float
    ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType();
    ft.setName("scaled_float");
    ft.setScalingFactor(0.1 + randomDouble() * 100);
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    final int numDocs = 1000;
    for (int i = 0; i < numDocs; ++i) {
        Document doc = new Document();
        double value = (randomDouble() * 2 - 1) * 10000;
        long scaledValue = Math.round(value * ft.getScalingFactor());
        double rounded = scaledValue / ft.getScalingFactor();
        doc.add(new LongPoint("scaled_float", scaledValue));
        doc.add(new DoublePoint("double", rounded));
        w.addDocument(doc);
    }
    final DirectoryReader reader = DirectoryReader.open(w);
    w.close();
    IndexSearcher searcher = newSearcher(reader);
    final int numQueries = 1000;
    for (int i = 0; i < numQueries; ++i) {
        Double l = randomBoolean() ? null : (randomDouble() * 2 - 1) * 10000;
        Double u = randomBoolean() ? null : (randomDouble() * 2 - 1) * 10000;
        boolean includeLower = randomBoolean();
        boolean includeUpper = randomBoolean();
        Query doubleQ = NumberFieldMapper.NumberType.DOUBLE.rangeQuery("double", l, u, includeLower, includeUpper, false);
        Query scaledFloatQ = ft.rangeQuery(l, u, includeLower, includeUpper, null);
        assertEquals(searcher.count(doubleQ), searcher.count(scaledFloatQ));
    }
    IOUtils.close(reader, dir);
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
private static void indexDoc(IndexWriter writer, FileBean t) throws Exception {
        Document doc = new Document();
        if (t.getContent() != null) {
            doc.add(new TextField(LuceneConstants.PATH, t.getFilepath(), Field.Store.YES));
            doc.add(new StringField(LuceneConstants.MODIFIED, UtilsTool.getDateStrByLastModified(t.getLastModified()), Field.Store.YES));
            doc.add(new TextField(LuceneConstants.CONTENT, t.getContent(), CommonConstants.IS_OPEN_CONTEXT ? Field.Store.YES : Field.Store.NO));
//            System.out.println("added to document:" + t.getFilepath());
            if (writer.getConfig().getOpenMode() == IndexWriterConfig.OpenMode.CREATE){
                writer.addDocument(doc);
            } else{
                writer.updateDocument(new Term(LuceneConstants.PATH, t.getFilepath()), doc);
            }
        }
    }
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
public void testRedisDirectoryWithJedisPool() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    //indexWriterConfig.setInfoStream(System.out);
    //indexWriterConfig.setRAMBufferSizeMB(2048);
    //LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
    //logByteSizeMergePolicy.setMinMergeMB(1);
    //logByteSizeMergePolicy.setMaxMergeMB(64);
    //logByteSizeMergePolicy.setMaxCFSSegmentSizeMB(64);
    //indexWriterConfig.setRAMBufferSizeMB(1024).setMergePolicy(logByteSizeMergePolicy).setUseCompoundFile(false);
    //GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
    //获取连接等待时间
    //genericObjectPoolConfig.setMaxWaitMillis(3000);
    //10s超时时间
    JedisPool jedisPool = new JedisPool(new JedisPoolConfig(), "localhost", 6379, Constants.TIME_OUT);
    RedisDirectory redisDirectory = new RedisDirectory(new JedisPoolStream(jedisPool));
    IndexWriter indexWriter = new IndexWriter(redisDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    redisDirectory.close();
    long end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(new RedisDirectory(new JedisStream("localhost",
            6379))));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool search consumes {}ms!", (end - start));
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
private static void addTicket(IndexWriter indexWriter, Ticket ticket) throws IOException {
	if (ticket.getSubject() == null || ticket.getId() == null){
		throw new IllegalArgumentException(ERROR_NULL_ARGUMENT);
	}
	
	Document doc = new Document();
	doc.add(new TextField(SUBJECT, ticket.getSubject(), Field.Store.YES));
	doc.add(new StringField(ID, ticket.getId(), Field.Store.YES));

	indexWriter.addDocument(doc);
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
/**
 * Index either a file or a directory tree.
 * 
 * @param writer
 * @param file
 * @throws IOException
 */
void indexDocs(IndexWriter writer, File file) throws IOException {

  /* Strip the absolute part of the path name from file name output */
  int scratchP = scratch.getCanonicalPath().length() + 1;

  /* do not try to index files that cannot be read */
  if (file.canRead()) {
    if (file.isDirectory()) {
      String[] files = file.list();
      // an IO error could occur
      if (files != null) {
        Arrays.sort(files);
        for (int i = 0; i < files.length; i++) {
          indexDocs(writer, new File(file, files[i]));
        }
      }
    } else {
      System.out.println("adding " + file.getCanonicalPath().substring(scratchP));
      try {
        writer.addDocument(FileDocument.Document(file));
      }
      // at least on windows, some temporary files raise this exception with
      // an "access denied" message
      // checking if the file can be read doesn't help
      catch (FileNotFoundException fnfe) { }
    }
  }
}
 

import org.apache.lucene.index.IndexWriter; //导入方法依赖的package包/类
private static void indexWithTrecContentSource(String dataDir, IndexWriter index) throws IOException {
    final long tStart = System.currentTimeMillis();
    int docCount = 0;
    TrecContentSource tcs = createTrecSource(dataDir);

    System.out.println("Start indexing ...");

    while (true) {
        DocData dd = new DocData();

        try {
            dd = tcs.getNextDocData(dd);
        } catch (Exception e) {
            if (e instanceof NoMoreDataException) {
                break;
            } else {
                System.err.println("Failed: " + e.getMessage());
                continue;
            }
        }

        Document doc = getDocumentFromDocData(dd);
        if (doc == null) {
            break;
        }

        docCount++;

        if ((docCount % 10000) == 0) {
            System.out.println("Total MB: " + tcs.getTotalBytesCount()/1000000 + " \t Docs: " + docCount + " (" + (System.currentTimeMillis() - tStart) / 1000.0 + " sec)");
        }

        index.addDocument(doc);
    }

    System.out.println("----- Fnished ----  (" + (System.currentTimeMillis() - tStart) / 1000.0 + " sec)");
    System.out.println("Total MB: " + tcs.getTotalBytesCount()/1000000);
    System.out.println("Total items: " + tcs.getTotalItemsCount());

}