Java FieldType.setStoreTermVectorPayloads方法代码示例

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testFieldTypeToTermVectorString() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(false);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat("with_positions_payloads", equalTo(ftOpts));
    AllFieldMapper.Builder builder = new AllFieldMapper.Builder(null);
    boolean exceptiontrown = false;
    try {
        TypeParsers.parseTermVector("", ftOpts, builder);
    } catch (MapperParsingException e) {
        exceptiontrown = true;
    }
    assertThat("TypeParsers.parseTermVector should accept string with_positions_payloads but does not.", exceptiontrown, equalTo(false));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testPayloadsWithoutPositions() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(false);
  customType.setStoreTermVectorPayloads(true);
  customType.setStoreTermVectorOffsets(random().nextBoolean());
  doc.add(new Field("field", "foo", customType));
  try {
    writer.addDocument(doc);
    fail();
  } catch (IllegalArgumentException expected) {
    // expected
  }
  writer.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeEmptyTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields fields = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(fields, null, flags, fields);
        outResponse.setExists(true);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("title", "the1 quick brown fox jumps over  the1 lazy dog", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testTermVectorStringGenerationWithoutPositions() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(false);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat(ftOpts, equalTo("with_offsets"));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = LuceneTestCase.newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("plaintext", "the1 quick brown fox jumps over  the1 lazy dog comment", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog comment", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
protected FieldType fieldType(Options options) {
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(options.positions);
  ft.setStoreTermVectorOffsets(options.offsets);
  ft.setStoreTermVectorPayloads(options.payloads);
  ft.freeze();
  return ft;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private Document newDocument() {
  Document doc = new Document();
  for (IndexOptions option : FieldInfo.IndexOptions.values()) {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    // turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setIndexOptions(option);
    doc.add(new Field(option.toString(), "", ft));
  }
  return doc;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testPayloads() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FieldType myFieldType = new FieldType(TextField.TYPE_NOT_STORED);
  myFieldType.setStoreTermVectors(true);
  myFieldType.setStoreTermVectorOffsets(true);
  myFieldType.setStoreTermVectorPositions(true);
  myFieldType.setStoreTermVectorPayloads(true);

  curOffset = 0;

  Token[] tokens = new Token[] {
    getToken("foxes"),
    getToken("can"),
    getToken("jump"),
    getToken("high")
  };

  Document doc = new Document();
  doc.add(new Field("field", new CannedTokenStream(tokens), myFieldType));
  writer.addDocument(doc);

  IndexReader reader = writer.getReader();
  writer.close();
  assertEquals(1, reader.numDocs());

  for(int i=0;i<2;i++) {
    // Do this twice, once passing true and then passing
    // false: they are entirely different code paths
    // under-the-hood:
    TokenStream ts = TokenSources.getTokenStream(reader.getTermVectors(0).terms("field"), i == 0);

    CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
    PositionIncrementAttribute posIncAtt = ts.getAttribute(PositionIncrementAttribute.class);
    OffsetAttribute offsetAtt = ts.getAttribute(OffsetAttribute.class);
    PayloadAttribute payloadAtt = ts.getAttribute(PayloadAttribute.class);

    for(Token token : tokens) {
      assertTrue(ts.incrementToken());
      assertEquals(token.toString(), termAtt.toString());
      assertEquals(token.getPositionIncrement(), posIncAtt.getPositionIncrement());
      assertEquals(token.getPayload(), payloadAtt.getPayload());
      assertEquals(token.startOffset(), offsetAtt.startOffset());
      assertEquals(token.endOffset(), offsetAtt.endOffset());
    }

    assertFalse(ts.incrementToken());
  }

  reader.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testDuelMemoryIndexCoreDirectoryWithArrayField() throws Exception {

    final String field_name = "text";
    MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
    if (random().nextBoolean()) {
      mockAnalyzer.setOffsetGap(random().nextInt(100));
    }
    //index into a random directory
    FieldType type = new FieldType(TextField.TYPE_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPayloads(false);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();

    Document doc = new Document();
    doc.add(new Field(field_name, "la la", type));
    doc.add(new Field(field_name, "foo bar foo bar foo", type));

    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), TEST_VERSION_CURRENT, mockAnalyzer));
    writer.updateDocument(new Term("id", "1"), doc);
    writer.commit();
    writer.close();
    DirectoryReader reader = DirectoryReader.open(dir);

    //Index document in Memory index
    MemoryIndex memIndex = new MemoryIndex(true);
    memIndex.addField(field_name, "la la", mockAnalyzer);
    memIndex.addField(field_name, "foo bar foo bar foo", mockAnalyzer);

    //compare term vectors
    Terms ramTv = reader.getTermVector(0, field_name);
    IndexReader memIndexReader = memIndex.createSearcher().getIndexReader();
    Terms memTv = memIndexReader.getTermVector(0, field_name);

    compareTermVectors(ramTv, memTv, field_name);
    memIndexReader.close();
    reader.close();
    dir.close();

  }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public synchronized static Field newField(Random random, String name, String value, FieldType type) {

    // Defeat any consumers that illegally rely on intern'd
    // strings (we removed this from Lucene a while back):
    name = new String(name);

    FieldType prevType = fieldToType.get(name);

    if (usually(random) || !type.indexed() || prevType != null) {
      // most of the time, don't modify the params
      if (prevType == null) {
        fieldToType.put(name, new FieldType(type));
      } else {
        type = mergeTermVectorOptions(type, prevType);
      }

      return new Field(name, value, type);
    }

    // TODO: once all core & test codecs can index
    // offsets, sometimes randomly turn on offsets if we are
    // already indexing positions...

    FieldType newType = new FieldType(type);
    if (!newType.stored() && random.nextBoolean()) {
      newType.setStored(true); // randomly store it
    }

    // Randomly turn on term vector options, but always do
    // so consistently for the same field name:
    if (!newType.storeTermVectors() && random.nextBoolean()) {
      newType.setStoreTermVectors(true);
      if (!newType.storeTermVectorPositions()) {
        newType.setStoreTermVectorPositions(random.nextBoolean());
        
        if (newType.storeTermVectorPositions()) {
          if (!newType.storeTermVectorPayloads() && !OLD_FORMAT_IMPERSONATION_IS_ACTIVE) {
            newType.setStoreTermVectorPayloads(random.nextBoolean());
          }
        }
      }
      
      if (!newType.storeTermVectorOffsets()) {
        newType.setStoreTermVectorOffsets(random.nextBoolean());
      }

      if (VERBOSE) {
        System.out.println("NOTE: LuceneTestCase: upgrade name=" + name + " type=" + newType);
      }
    }
    newType.freeze();
    fieldToType.put(name, newType);

    // TODO: we need to do this, but smarter, ie, most of
    // the time we set the same value for a given field but
    // sometimes (rarely) we change it up:
    /*
    if (newType.omitNorms()) {
      newType.setOmitNorms(random.nextBoolean());
    }
    */
    
    return new Field(name, value, newType);
  }
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private static FieldType _fieldTypeFrom(final IndexDocumentStandardFieldType fieldType,
										final boolean stored,
										final IndexDocumentFieldIndexOptions options) {
	FieldType outFieldType = new FieldType();
	
	// [1] - Set if the field is stored or not
	outFieldType.setStored(stored);
	
	// [2] - Set the options: indexed, tokenized, etc
	if (options != null) {
		outFieldType.setIndexed(true);
		outFieldType.setOmitNorms(options.isOmitingNorms());
		outFieldType.setTokenized(options.isTokenized());
		if (options.getTermVectorsStoring() != null) {
			outFieldType.setStoreTermVectors(options.getTermVectorsStoring().isEnabled());
			if (options.getTermVectorsStoring().isEnabled()) {
				outFieldType.setStoreTermVectorOffsets(options.getTermVectorsStoring().isIncludeTermOffsets());
				outFieldType.setStoreTermVectorPositions(options.getTermVectorsStoring().isIncludeTermPositions());
				outFieldType.setStoreTermVectorPayloads(options.getTermVectorsStoring().isIncludeTermPayLoads());
			}
		}
	}
	// [3] - Set the lucene's FieldType's numericType
	switch(fieldType) {
	case Double:
		outFieldType.setNumericType(FieldType.NumericType.DOUBLE);
		break;
	case Float:
		outFieldType.setNumericType(FieldType.NumericType.FLOAT);
		break;
	case Int:
		outFieldType.setNumericType(FieldType.NumericType.INT);
		break;
	case Long:
		outFieldType.setNumericType(FieldType.NumericType.LONG);
		break;
	case String:
		// nothing
		break;
	case Text:
		// nothing
		break;
	default:
		throw new IllegalArgumentException(fieldType + "is NOT a supported type");
	}
	return outFieldType;
}