Java TextField.TYPE_NOT_STORED属性代码示例

@Override
protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
    // we know its low level reader, and matching docId, since that's how we call the highlighter with
    SourceLookup sourceLookup = searchContext.lookup().source();
    sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);

    List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
    if (values.isEmpty()) {
        return EMPTY_FIELDS;
    }
    Field[] fields = new Field[values.size()];
    for (int i = 0; i < values.size(); i++) {
        fields[i] = new Field(mapper.fieldType().name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
    }
    return fields;
}
 

public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}
 

@Override
protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
    // we know its low level reader, and matching docId, since that's how we call the highlighter with
    SourceLookup sourceLookup = searchContext.lookup().source();
    sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);

    List<Object> values = sourceLookup.extractRawValues(hitContext.getSourcePath(mapper.fieldType().names().fullName()));
    if (values.isEmpty()) {
        return EMPTY_FIELDS;
    }
    Field[] fields = new Field[values.size()];
    for (int i = 0; i < values.size(); i++) {
        fields[i] = new Field(mapper.fieldType().names().indexName(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
    }
    return fields;
}
 

private Document getDocument(File file) throws IOException {
	Document document = new Document();

	// index file contents
	Field contentField = new Field(LuceneConstants.CONTENTS, new FileReader(file), TextField.TYPE_NOT_STORED);
	// index file name
	Field fileNameField = new Field(LuceneConstants.FILE_NAME, file.getName(), TextField.TYPE_STORED);
	// index file path
	Field filePathField = new Field(LuceneConstants.FILE_PATH, file.getCanonicalPath(), TextField.TYPE_STORED);

	document.add(contentField);
	document.add(fileNameField);
	document.add(filePathField);

	return document;
}
 

public void testPayloadsWithoutPositions() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(false);
  customType.setStoreTermVectorPayloads(true);
  customType.setStoreTermVectorOffsets(random().nextBoolean());
  doc.add(new Field("field", "foo", customType));
  try {
    writer.addDocument(doc);
    fail();
  } catch (IllegalArgumentException expected) {
    // expected
  }
  writer.close();
  dir.close();
}
 

public void testLegalbutVeryLargeOffsets() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  Token t1 = new Token("foo", 0, Integer.MAX_VALUE-500);
  if (random().nextBoolean()) {
    t1.setPayload(new BytesRef("test"));
  }
  Token t2 = new Token("foo", Integer.MAX_VALUE-500, Integer.MAX_VALUE);
  TokenStream tokenStream = new CannedTokenStream(
      new Token[] { t1, t2 }
  );
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  // store some term vectors for the checkindex cross-check
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  ft.setStoreTermVectorOffsets(true);
  Field field = new Field("foo", tokenStream, ft);
  doc.add(field);
  iw.addDocument(doc);
  iw.close();
  dir.close();
}
 

public void testMaxThreadPriority() throws IOException {
  int pri = Thread.currentThread().getPriority();
  try {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setMaxBufferedDocs(2)
                               .setMergePolicy(newLogMergePolicy());
    ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
    IndexWriter iw = new IndexWriter(dir, conf);
    Document document = new Document();
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setStoreTermVectors(true);
    document.add(newField("tvtest", "a b c", customType));
    Thread.currentThread().setPriority(Thread.MAX_PRIORITY);
    for(int i=0;i<4;i++)
      iw.addDocument(document);
    iw.close();
    dir.close();
  } finally {
    Thread.currentThread().setPriority(pri);
  }
}
 

private void doTestMixedPostings(Codec codec) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(codec);
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  // turn on vectors for the checkindex cross-check
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  Field idField = new Field("id", "", ft);
  Field dateField = new Field("date", "", ft);
  doc.add(idField);
  doc.add(dateField);
  for (int i = 0; i < 100; i++) {
    idField.setStringValue(Integer.toString(random().nextInt(50)));
    dateField.setStringValue(Integer.toString(random().nextInt(100)));
    iw.addDocument(doc);
  }
  iw.close();
  dir.close(); // checkindex
}
 

@Override
protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
    // we know its low level reader, and matching docId, since that's how we call the highlighter with
    SourceLookup sourceLookup = searchContext.lookup().source();
    sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);

    List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
    Field[] fields = new Field[values.size()];
    for (int i = 0; i < values.size(); i++) {
        fields[i] = new Field(mapper.fieldType().name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
    }
    return fields;
}
 

@Override
protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
    // we know its low level reader, and matching docId, since that's how we call the highlighter with
    SourceLookup sourceLookup = searchContext.lookup().source();
    sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);

    List<Object> values = sourceLookup.extractRawValues(hitContext.getSourcePath(mapper.fieldType().names().fullName()));
    Field[] fields = new Field[values.size()];
    for (int i = 0; i < values.size(); i++) {
        fields[i] = new Field(mapper.fieldType().names().indexName(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
    }
    return fields;
}
 

public void test() throws Exception {
  BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPositions"));
  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
  }
  
  IndexWriter w = new IndexWriter(dir,
      new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
      .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setRAMBufferSizeMB(256.0)
      .setMergeScheduler(new ConcurrentMergeScheduler())
      .setMergePolicy(newLogMergePolicy(false, 10))
      .setOpenMode(IndexWriterConfig.OpenMode.CREATE));

  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogByteSizeMergePolicy) {
   // 1 petabyte:
   ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
  }

  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setOmitNorms(true);
  Field field = new Field("field", new MyTokenStream(), ft);
  doc.add(field);
  
  final int numDocs = (Integer.MAX_VALUE / 26) + 1;
  for (int i = 0; i < numDocs; i++) {
    w.addDocument(doc);
    if (VERBOSE && i % 100000 == 0) {
      System.out.println(i + " of " + numDocs + "...");
    }
  }
  w.forceMerge(1);
  w.close();
  dir.close();
}
 

/**
 * Test adding two fields with the same name, one indexed
 * the other stored only. The omitNorms and omitTermFreqAndPositions setting
 * of the stored field should not affect the indexed one (LUCENE-1590)
 */
public void testLUCENE_1590() throws Exception {
  Document doc = new Document();
  // f1 has no norms
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setOmitNorms(true);
  FieldType customType2 = new FieldType();
  customType2.setStored(true);
  doc.add(newField("f1", "v1", customType));
  doc.add(newField("f1", "v2", customType2));
  // f2 has no TF
  FieldType customType3 = new FieldType(TextField.TYPE_NOT_STORED);
  customType3.setIndexOptions(IndexOptions.DOCS_ONLY);
  Field f = newField("f2", "v1", customType3);
  doc.add(f);
  doc.add(newField("f2", "v2", customType2));

  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  writer.addDocument(doc);
  writer.forceMerge(1); // be sure to have a single segment
  writer.close();

  TestUtil.checkIndex(dir);

  SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(dir));
  FieldInfos fi = reader.getFieldInfos();
  // f1
  assertFalse("f1 should have no norms", fi.fieldInfo("f1").hasNorms());
  assertEquals("omitTermFreqAndPositions field bit should not be set for f1", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").getIndexOptions());
  // f2
  assertTrue("f2 should have norms", fi.fieldInfo("f2").hasNorms());
  assertEquals("omitTermFreqAndPositions field bit should be set for f2", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").getIndexOptions());
  reader.close();
}
 

@Test
public void testStressPerFieldCodec() throws IOException {
  Directory dir = newDirectory(random());
  final int docsPerRound = 97;
  int numRounds = atLeast(1);
  for (int i = 0; i < numRounds; i++) {
    int num = TestUtil.nextInt(random(), 30, 60);
    IndexWriterConfig config = newIndexWriterConfig(random(),
        TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    config.setOpenMode(OpenMode.CREATE_OR_APPEND);
    IndexWriter writer = newWriter(dir, config);
    for (int j = 0; j < docsPerRound; j++) {
      final Document doc = new Document();
      for (int k = 0; k < num; k++) {
        FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
        customType.setTokenized(random().nextBoolean());
        customType.setOmitNorms(random().nextBoolean());
        Field field = newField("" + k, TestUtil
            .randomRealisticUnicodeString(random(), 128), customType);
        doc.add(field);
      }
      writer.addDocument(doc);
    }
    if (random().nextBoolean()) {
      writer.forceMerge(1);
    }
    writer.commit();
    assertEquals((i + 1) * docsPerRound, writer.maxDoc());
    writer.close();
  }
  dir.close();
}
 

public void testSlopWithHoles() throws Exception {  
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setOmitNorms(true);
  Field f = new Field("lyrics", "", customType);
  Document doc = new Document();
  doc.add(f);
  f.setStringValue("drug drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy druggy drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy drug druggy drug");
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);
  
  PhraseQuery pq = new PhraseQuery();
  // "drug the drug"~1
  pq.add(new Term("lyrics", "drug"), 1);
  pq.add(new Term("lyrics", "drug"), 4);
  pq.setSlop(0);
  assertEquals(0, is.search(pq, 4).totalHits);
  pq.setSlop(1);
  assertEquals(3, is.search(pq, 4).totalHits);
  pq.setSlop(2);
  assertEquals(4, is.search(pq, 4).totalHits);
  ir.close();
  dir.close();
}
 

protected FieldType fieldType(Options options) {
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(options.positions);
  ft.setStoreTermVectorOffsets(options.offsets);
  ft.setStoreTermVectorPayloads(options.payloads);
  ft.freeze();
  return ft;
}