Java FieldType.setStoreTermVectorOffsets方法代码示例

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testVectorHighlighter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, notNullValue());
    assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testFieldTypeToTermVectorString() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(false);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat("with_positions_payloads", equalTo(ftOpts));
    AllFieldMapper.Builder builder = new AllFieldMapper.Builder(null);
    boolean exceptiontrown = false;
    try {
        TypeParsers.parseTermVector("", ftOpts, builder);
    } catch (MapperParsingException e) {
        exceptiontrown = true;
    }
    assertThat("TypeParsers.parseTermVector should accept string with_positions_payloads but does not.", exceptiontrown, equalTo(false));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public LuceneUtil(JasperReportsContext jasperReportsContext, boolean isCaseSensitive, boolean isWholeWordsOnly, boolean removeAccents) {
	this.isCaseSensitive = isCaseSensitive;
	this.isWholeWordsOnly = isWholeWordsOnly;
	this.removeAccents = removeAccents;

	this.noneSelector = JRStyledTextAttributeSelector.getNoneSelector(jasperReportsContext);
	this.styledTextUtil = JRStyledTextUtil.getInstance(jasperReportsContext);
	
	fieldType = new FieldType();
	fieldType.setIndexed(true);
	fieldType.setTokenized(true);
	fieldType.setStored(true);
	fieldType.setStoreTermVectors(true);
	fieldType.setStoreTermVectorPositions(true);
	fieldType.setStoreTermVectorOffsets(true);
	fieldType.freeze();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void populatePerformanceIndex(Analyzer analyzer) throws IOException {
  indexWriter.deleteAll();
  indexWriter.commit();

  FieldType ft = new FieldType(TextField.TYPE_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  int docs = 1000;
  Random random = random();
  for (int i = 0; i < docs; i++) {
    boolean b = random.nextBoolean();
    Document doc = new Document();
    doc.add(new Field(textFieldName, createRandomString(random), ft));
    doc.add(new Field(categoryFieldName, b ? "technology" : "politics", ft));
    doc.add(new Field(booleanFieldName, String.valueOf(b), ft));
    indexWriter.addDocument(doc, analyzer);
  }
  indexWriter.commit();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testPayloadsWithoutPositions() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(false);
  customType.setStoreTermVectorPayloads(true);
  customType.setStoreTermVectorOffsets(random().nextBoolean());
  doc.add(new Field("field", "foo", customType));
  try {
    writer.addDocument(doc);
    fail();
  } catch (IllegalArgumentException expected) {
    // expected
  }
  writer.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testBogusTermVectors() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  Field field = new Field("foo", "", ft);
  field.setTokenStream(new CannedTokenStream(
      new Token("bar", 5, 10), new Token("bar", 1, 4)
  ));
  doc.add(field);
  iw.addDocument(doc);
  iw.close();
  dir.close(); // checkindex
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void checkTokens(Token[] tokens) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);
   
    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(tokens), ft));
    riw.addDocument(doc);
    success = true;
  } finally {
    if (success) {
      IOUtils.close(riw, dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeEmptyTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields fields = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(fields, null, flags, fields);
        outResponse.setExists(true);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("title", "the1 quick brown fox jumps over  the1 lazy dog", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testTermVectorStringGenerationWithoutPositions() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(false);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat(ftOpts, equalTo("with_offsets"));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = LuceneTestCase.newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("plaintext", "the1 quick brown fox jumps over  the1 lazy dog comment", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog comment", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testDeadlock() throws Exception {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                              .setMaxBufferedDocs(2));
  Document doc = new Document();

  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(true);
  customType.setStoreTermVectorOffsets(true);
  
  doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType));
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.commit();
  // index has 2 segments

  Directory dir2 = newDirectory();
  IndexWriter writer2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random())));
  writer2.addDocument(doc);
  writer2.close();

  IndexReader r1 = DirectoryReader.open(dir2);
  writer.addIndexes(r1, r1);
  writer.close();

  IndexReader r3 = DirectoryReader.open(dir);
  assertEquals(5, r3.numDocs());
  r3.close();

  r1.close();

  dir2.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  indexWriter = new RandomIndexWriter(random(), dir);
  textFieldName = "text";
  categoryFieldName = "cat";
  booleanFieldName = "bool";
  ft = new FieldType(TextField.TYPE_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public DocState(boolean useDocValues) {
  doc = new Document();
  
  title = new StringField("title", "", Field.Store.NO);
  doc.add(title);

  FieldType ft = new FieldType(TextField.TYPE_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  
  titleTokenized = new Field("titleTokenized", "", ft);
  doc.add(titleTokenized);

  body = new Field("body", "", ft);
  doc.add(body);

  id = new StringField("docid", "", Field.Store.YES);
  doc.add(id);

  date = new StringField("date", "", Field.Store.YES);
  doc.add(date);

  if (useDocValues) {
    titleDV = new SortedDocValuesField("titleDV", new BytesRef());
    doc.add(titleDV);
  } else {
    titleDV = null;
  }
}