Java FieldType.setStoreTermVectors方法代码示例

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testVectorHighlighter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, notNullValue());
    assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testFieldTypeToTermVectorString() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(false);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat("with_positions_payloads", equalTo(ftOpts));
    AllFieldMapper.Builder builder = new AllFieldMapper.Builder(null);
    boolean exceptiontrown = false;
    try {
        TypeParsers.parseTermVector("", ftOpts, builder);
    } catch (MapperParsingException e) {
        exceptiontrown = true;
    }
    assertThat("TypeParsers.parseTermVector should accept string with_positions_payloads but does not.", exceptiontrown, equalTo(false));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public LuceneUtil(JasperReportsContext jasperReportsContext, boolean isCaseSensitive, boolean isWholeWordsOnly, boolean removeAccents) {
	this.isCaseSensitive = isCaseSensitive;
	this.isWholeWordsOnly = isWholeWordsOnly;
	this.removeAccents = removeAccents;

	this.noneSelector = JRStyledTextAttributeSelector.getNoneSelector(jasperReportsContext);
	this.styledTextUtil = JRStyledTextUtil.getInstance(jasperReportsContext);
	
	fieldType = new FieldType();
	fieldType.setIndexed(true);
	fieldType.setTokenized(true);
	fieldType.setStored(true);
	fieldType.setStoreTermVectors(true);
	fieldType.setStoreTermVectorPositions(true);
	fieldType.setStoreTermVectorOffsets(true);
	fieldType.freeze();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testLegalbutVeryLargeOffsets() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  Token t1 = new Token("foo", 0, Integer.MAX_VALUE-500);
  if (random().nextBoolean()) {
    t1.setPayload(new BytesRef("test"));
  }
  Token t2 = new Token("foo", Integer.MAX_VALUE-500, Integer.MAX_VALUE);
  TokenStream tokenStream = new CannedTokenStream(
      new Token[] { t1, t2 }
  );
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  // store some term vectors for the checkindex cross-check
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  ft.setStoreTermVectorOffsets(true);
  Field field = new Field("foo", tokenStream, ft);
  doc.add(field);
  iw.addDocument(doc);
  iw.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void checkTokens(Token[] field1, Token[] field2) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);
   
    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(field1), ft));
    doc.add(new Field("body", new CannedTokenStream(field2), ft));
    riw.addDocument(doc);
    riw.close();
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
@Override
public boolean filter(FullConcept concept) {
    //index
    Document doc = new Document();

    /** The customized field type for contents field */
    FieldType contentFieldType = new FieldType();
    contentFieldType.setIndexed(true);
    contentFieldType.setStored(true);
    contentFieldType.setStoreTermVectors(true);
    contentFieldType.setTokenized(true);

    doc.add(new Field("contents", concept.getTitle() + "\n" + concept.getPlainContent(), contentFieldType));
    doc.add(new StringField("id", Integer.toString(concept.getId()), Field.Store.YES));
    doc.add(new StringField("outId", concept.getOutId(), Field.Store.YES));
    doc.add(new Field("title", concept.getTitle(), contentFieldType));

    try {
        writer.addDocument(doc);
    } catch (IOException e) {
        e.printStackTrace();
    }

    return true;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException
{
    Document doc = new Document();
    FieldType customType5 = new FieldType(TextField.TYPE_STORED);
    customType5.setStoreTermVectors(true);
    FieldType customType6 = new FieldType(TextField.TYPE_STORED);
    customType6.setStoreTermVectors(true);
    customType6.setStoreTermVectorOffsets(true);
    FieldType customType7 = new FieldType(TextField.TYPE_STORED);
    customType7.setStoreTermVectors(true);
    customType7.setStoreTermVectorPositions(true);
    FieldType customType8 = new FieldType(TextField.TYPE_STORED);
    customType8.setStoreTermVectors(true);
    customType8.setStoreTermVectorOffsets(true);
    customType8.setStoreTermVectorPositions(true);
    doc.add(newTextField("tvnot", "tvnot", Field.Store.YES));
    doc.add(newField("termvector","termvector",customType5));
    doc.add(newField("tvoffset","tvoffset", customType6));
    doc.add(newField("tvposition","tvposition", customType7));
    doc.add(newField("tvpositionoffset","tvpositionoffset", customType8));
    
    writer.addDocument(doc);
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void index(Item item) throws IOException {
	String id = item.getId();
   	String text = item.getText();
   	
   	long publicationTIme = item.getPublicationTime();
   	
   	Document document = new Document();
	
	Field idField = new StringField("id", id, Store.YES);
   	document.add(idField);
	
	FieldType fieldType = new FieldType();
	fieldType.setStored(true);
	fieldType.setIndexed(true);
	fieldType.setStoreTermVectors(true);
	document.add(new Field("text", text, fieldType));
	
	document.add(new LongField("publicationTIme", publicationTIme, LongField.TYPE_STORED));
	if(iwriter != null) {
		iwriter.addDocument(document);
	}
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testMaxThreadPriority() throws IOException {
  int pri = Thread.currentThread().getPriority();
  try {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setMaxBufferedDocs(2)
                               .setMergePolicy(newLogMergePolicy());
    ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
    IndexWriter iw = new IndexWriter(dir, conf);
    Document document = new Document();
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setStoreTermVectors(true);
    document.add(newField("tvtest", "a b c", customType));
    Thread.currentThread().setPriority(Thread.MAX_PRIORITY);
    for(int i=0;i<4;i++)
      iw.addDocument(document);
    iw.close();
    dir.close();
  } finally {
    Thread.currentThread().setPriority(pri);
  }
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public static Document createDocument(int n, String indexName, int numFields) {
  StringBuilder sb = new StringBuilder();
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(true);
  customType.setStoreTermVectorOffsets(true);

  FieldType customType1 = new FieldType(StringField.TYPE_STORED);
  customType1.setStoreTermVectors(true);
  customType1.setStoreTermVectorPositions(true);
  customType1.setStoreTermVectorOffsets(true);

  final Document doc = new Document();
  doc.add(new Field("id", Integer.toString(n), customType1));
  doc.add(new Field("indexname", indexName, customType1));
  sb.append("a");
  sb.append(n);
  doc.add(new Field("field1", sb.toString(), customType));
  sb.append(" b");
  sb.append(n);
  for (int i = 1; i < numFields; i++) {
    doc.add(new Field("field" + (i + 1), sb.toString(), customType));
  }
  return doc;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeEmptyTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields fields = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(fields, null, flags, fields);
        outResponse.setExists(true);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("title", "the1 quick brown fox jumps over  the1 lazy dog", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testTermVectorStringGenerationWithoutPositions() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(false);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat(ftOpts, equalTo("with_offsets"));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void index() throws CorruptIndexException,
    LockObtainFailedException, IOException {
    Directory dir = FSDirectory.open(indexDirectory);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_41,StandardAnalyzer.STOP_WORDS_SET);
    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_41, analyzer);
    if (indexDirectory.exists()) {
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    } else {
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    }
    IndexWriter writer = new IndexWriter(dir, iwc);
    for (File f : sourceDirectory.listFiles()) {
        System.out.println("Indexing Document  "+f.getName());
        Document doc = new Document();
        FieldType fieldType = new FieldType();
        fieldType.setIndexed(true);
        fieldType.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
        fieldType.setStored(true);
        fieldType.setStoreTermVectors(true);
        fieldType.setTokenized(true);
        Field contentField = new Field(fieldName, ExtractText(f), fieldType);
        doc.add(contentField);
        doc.add(new TextField("FileName", f.getName(), Field.Store.YES));
        doc.add(new TextField("FilePath",f.getCanonicalPath(),Field.Store.YES));
        writer.addDocument(doc);
    }
    writer.close();
}