Java FieldType.setOmitNorms方法代码示例

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private static void addDoc(RandomIndexWriter iw, int i) throws Exception {
  Document d = new Document();
  Field f;
  int scoreAndID = i + 1;

  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setTokenized(false);
  customType.setOmitNorms(true);
  
  f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
  d.add(f);

  FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
  customType2.setOmitNorms(true);
  f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
  d.add(f);

  f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring
  d.add(f);

  f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring
  d.add(f);

  iw.addDocument(d);
  log("added: " + d);
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
/** make sure we can retrieve when norms are disabled */
public void testNoNorms() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setOmitNorms(true);
  ft.freeze();
  doc.add(newField("foo", "bar", ft));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);
  
  for (Similarity sim : sims) {
    is.setSimilarity(sim);
    BooleanQuery query = new BooleanQuery(true);
    query.add(new TermQuery(new Term("foo", "bar")), BooleanClause.Occur.SHOULD);
    assertEquals(1, is.search(query, 10).totalHits);
  }
  ir.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public static Document createDocument(int n, int numFields) {
  StringBuilder sb = new StringBuilder();
  Document doc = new Document();
  sb.append("a");
  sb.append(n);
  FieldType customType2 = new FieldType(TextField.TYPE_STORED);
  customType2.setTokenized(false);
  customType2.setOmitNorms(true);
  FieldType customType3 = new FieldType();
  customType3.setStored(true);
  doc.add(new TextField("field1", sb.toString(), Field.Store.YES));
  doc.add(new Field("fielda", sb.toString(), customType2));
  doc.add(new Field("fieldb", sb.toString(), customType3));
  sb.append(" b");
  sb.append(n);
  for (int i = 1; i < numFields; i++) {
    doc.add(new TextField("field" + (i+1), sb.toString(), Field.Store.YES));
  }
  return doc;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private FieldType createFieldType() {
	FieldType ft = new FieldType();
	ft.setIndexed(true);
	ft.setTokenized(true);
	ft.setOmitNorms(true);
	ft.freeze();
	return ft;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private static FieldType getFieldType() {
	FieldType ft = new FieldType();
	ft.setIndexed(true);
	ft.setTokenized(true);
	ft.setOmitNorms(true);
	ft.freeze();
	return ft;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void openExampleIndex() throws IOException {
  //Create a RAM-based index from our test data file
  RAMDirectory rd = new RAMDirectory();
  IndexWriterConfig iwConfig = new IndexWriterConfig(Version.LUCENE_CURRENT, analyzer);
  IndexWriter writer = new IndexWriter(rd, iwConfig);
  InputStream dataIn = getServletContext().getResourceAsStream("/WEB-INF/data.tsv");
  BufferedReader br = new BufferedReader(new InputStreamReader(dataIn, StandardCharsets.UTF_8));
  String line = br.readLine();
  final FieldType textNoNorms = new FieldType(TextField.TYPE_STORED);
  textNoNorms.setOmitNorms(true);
  while (line != null) {
    line = line.trim();
    if (line.length() > 0) {
      //parse row and create a document
      StringTokenizer st = new StringTokenizer(line, "\t");
      Document doc = new Document();
      doc.add(new Field("location", st.nextToken(), textNoNorms));
      doc.add(new Field("salary", st.nextToken(), textNoNorms));
      doc.add(new Field("type", st.nextToken(), textNoNorms));
      doc.add(new Field("description", st.nextToken(), textNoNorms));
      writer.addDocument(doc);
    }
    line = br.readLine();
  }
  writer.close();

  //open searcher
  // this example never closes it reader!
  IndexReader reader = DirectoryReader.open(rd);
  searcher = new IndexSearcher(reader);
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
@Override
protected FieldType getTextFieldType() {
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  ft.setOmitNorms(true);

  return ft;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public IndexableField createField(SchemaField field, Object value, float boost) {
  final boolean indexed = field.indexed();
  final boolean stored = field.stored();
  final boolean docValues = field.hasDocValues();

  if (!indexed && !stored && !docValues) {
    if (log.isTraceEnabled())
      log.trace("Ignoring unindexed/unstored field: " + field);
    return null;
  }
  final Integer intValue = stringValueToIntValue(value.toString());
  if (intValue == null || intValue.equals(DEFAULT_VALUE))
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown value for enum field: " + value.toString());

  String intAsString =  intValue.toString();
  final FieldType newType = new FieldType();

  newType.setIndexed(field.indexed());
  newType.setTokenized(field.isTokenized());
  newType.setStored(field.stored());
  newType.setOmitNorms(field.omitNorms());
  newType.setIndexOptions(getIndexOptions(field, intAsString));
  newType.setStoreTermVectors(field.storeTermVector());
  newType.setStoreTermVectorOffsets(field.storeTermOffsets());
  newType.setStoreTermVectorPositions(field.storeTermPositions());
  newType.setNumericType(FieldType.NumericType.INT);
  newType.setNumericPrecisionStep(DEFAULT_PRECISION_STEP);

  final org.apache.lucene.document.Field f;
  f = new org.apache.lucene.document.IntField(field.getName(), intValue.intValue(), newType);

  f.setBoost(boost);
  return f;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
/**
 * Tests various combinations of omitNorms=true/false, the field not existing at all,
 * ensuring that only omitNorms is 'viral'.
 * Internally checks that MultiNorms.norms() is consistent (returns the same bytes)
 * as the fully merged equivalent.
 */
public void testOmitNormsCombos() throws IOException {
  // indexed with norms
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  Field norms = new Field("foo", "a", customType);
  // indexed without norms
  FieldType customType1 = new FieldType(TextField.TYPE_STORED);
  customType1.setOmitNorms(true);
  Field noNorms = new Field("foo", "a", customType1);
  // not indexed, but stored
  FieldType customType2 = new FieldType();
  customType2.setStored(true);
  Field noIndex = new Field("foo", "a", customType2);
  // not indexed but stored, omitNorms is set
  FieldType customType3 = new FieldType();
  customType3.setStored(true);
  customType3.setOmitNorms(true);
  Field noNormsNoIndex = new Field("foo", "a", customType3);
  // not indexed nor stored (doesnt exist at all, we index a different field instead)
  Field emptyNorms = new Field("bar", "a", customType);
  
  assertNotNull(getNorms("foo", norms, norms));
  assertNull(getNorms("foo", norms, noNorms));
  assertNotNull(getNorms("foo", norms, noIndex));
  assertNotNull(getNorms("foo", norms, noNormsNoIndex));
  assertNotNull(getNorms("foo", norms, emptyNorms));
  assertNull(getNorms("foo", noNorms, noNorms));
  assertNull(getNorms("foo", noNorms, noIndex));
  assertNull(getNorms("foo", noNorms, noNormsNoIndex));
  assertNull(getNorms("foo", noNorms, emptyNorms));
  assertNull(getNorms("foo", noIndex, noIndex));
  assertNull(getNorms("foo", noIndex, noNormsNoIndex));
  assertNull(getNorms("foo", noIndex, emptyNorms));
  assertNull(getNorms("foo", noNormsNoIndex, noNormsNoIndex));
  assertNull(getNorms("foo", noNormsNoIndex, emptyNorms));
  assertNull(getNorms("foo", emptyNorms, emptyNorms));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private static Document makeDocument(String docText) {
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setOmitNorms(true);
  Field f = new Field("f", docText, customType);
  doc.add(f);
  return doc;
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testSlopWithHoles() throws Exception {  
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setOmitNorms(true);
  Field f = new Field("lyrics", "", customType);
  Document doc = new Document();
  doc.add(f);
  f.setStringValue("drug drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy druggy drug");
  iw.addDocument(doc);
  f.setStringValue("drug druggy drug druggy drug");
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  IndexSearcher is = newSearcher(ir);
  
  PhraseQuery pq = new PhraseQuery();
  // "drug the drug"~1
  pq.add(new Term("lyrics", "drug"), 1);
  pq.add(new Term("lyrics", "drug"), 4);
  pq.setSlop(0);
  assertEquals(0, is.search(pq, 4).totalHits);
  pq.setSlop(1);
  assertEquals(3, is.search(pq, 4).totalHits);
  pq.setSlop(2);
  assertEquals(4, is.search(pq, 4).totalHits);
  ir.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
  final FieldType ft = new FieldType(TextField.TYPE_STORED);
  ft.setStoreTermVectors(fieldInfo.hasVectors());
  ft.setIndexed(fieldInfo.isIndexed());
  ft.setOmitNorms(fieldInfo.omitsNorms());
  ft.setIndexOptions(fieldInfo.getIndexOptions());
  doc.add(new Field(fieldInfo.name, value, ft));
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void test() throws Exception {
  BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPositions"));
  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
  }
  
  IndexWriter w = new IndexWriter(dir,
      new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
      .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
      .setRAMBufferSizeMB(256.0)
      .setMergeScheduler(new ConcurrentMergeScheduler())
      .setMergePolicy(newLogMergePolicy(false, 10))
      .setOpenMode(IndexWriterConfig.OpenMode.CREATE));

  MergePolicy mp = w.getConfig().getMergePolicy();
  if (mp instanceof LogByteSizeMergePolicy) {
   // 1 petabyte:
   ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
  }

  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setOmitNorms(true);
  Field field = new Field("field", new MyTokenStream(), ft);
  doc.add(field);
  
  final int numDocs = (Integer.MAX_VALUE / 26) + 1;
  for (int i = 0; i < numDocs; i++) {
    w.addDocument(doc);
    if (VERBOSE && i % 100000 == 0) {
      System.out.println(i + " of " + numDocs + "...");
    }
  }
  w.forceMerge(1);
  w.close();
  dir.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
public void testNoNrmFile() throws Throwable {
  Directory ram = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
                                              .setMaxBufferedDocs(3)
                                              .setMergePolicy(newLogMergePolicy()));
  LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
  lmp.setMergeFactor(2);
  lmp.setNoCFSRatio(0.0);
  Document d = new Document();

  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setOmitNorms(true);
  Field f1 = newField("f1", "This field has no norms", customType);
  d.add(f1);

  for (int i = 0; i < 30; i++) {
    writer.addDocument(d);
  }

  writer.commit();

  assertNoNrm(ram);
      
  // force merge
  writer.forceMerge(1);
  // flush
  writer.close();

  assertNoNrm(ram);
  ram.close();
}
 

import org.apache.lucene.document.FieldType; //导入方法依赖的package包/类
private void addCommonDocumentFields(Document document, FeatureIndexEntry entry, final Long featureFileId) {
    document.add(new SortedStringField(FeatureIndexFields.FEATURE_ID.getFieldName(), entry.getFeatureId()));

    FieldType fieldType = new FieldType();
    fieldType.setOmitNorms(true);
    fieldType.setIndexOptions(IndexOptions.DOCS);
    fieldType.setStored(true);
    fieldType.setTokenized(false);
    fieldType.setDocValuesType(DocValuesType.SORTED);
    fieldType.freeze();
    Field field = new Field(FeatureIndexFields.CHROMOSOME_ID.getFieldName(), entry.getChromosome() != null ?
            new BytesRef(entry.getChromosome().getId().toString()) : new BytesRef(""), fieldType);
    document.add(field);
    document.add(new SortedStringField(FeatureIndexFields.CHROMOSOME_NAME.getFieldName(),
            entry.getChromosome().getName(), true));

    document.add(new SortedIntPoint(FeatureIndexFields.START_INDEX.getFieldName(), entry.getStartIndex()));
    document.add(new StoredField(FeatureIndexFields.START_INDEX.getFieldName(), entry.getStartIndex()));
    document.add(new SortedDocValuesField(FeatureIndexFields.START_INDEX.getGroupName(),
            new BytesRef(entry.getStartIndex().toString())));

    document.add(new SortedIntPoint(FeatureIndexFields.END_INDEX.getFieldName(), entry.getEndIndex()));
    document.add(new StoredField(FeatureIndexFields.END_INDEX.getFieldName(), entry.getEndIndex()));
    document.add(new SortedDocValuesField(FeatureIndexFields.END_INDEX.getGroupName(),
            new BytesRef(entry.getStartIndex().toString())));

    document.add(new StringField(FeatureIndexFields.FEATURE_TYPE.getFieldName(),
            entry.getFeatureType() != null ? entry.getFeatureType().getFileValue() : "", Field.Store.YES));
    document.add(new StringField(FeatureIndexFields.FILE_ID.getFieldName(), featureFileId.toString(),
            Field.Store.YES));

    document.add(new StringField(FeatureIndexFields.FEATURE_NAME.getFieldName(),
            entry.getFeatureName() != null ? entry.getFeatureName().toLowerCase() : "", Field.Store.YES));
    document.add(new SortedDocValuesField(FeatureIndexFields.FEATURE_NAME.getFieldName(),
            new BytesRef(entry.getFeatureName() != null ? entry.getFeatureName() : "")));

    document.add(new SortedSetDocValuesFacetField(FeatureIndexFields.CHR_ID.getFieldName(),
            entry.getChromosome().getId().toString()));

    document.add(new SortedStringField(FeatureIndexFields.UID.getFieldName(), entry.getUuid().toString()));
    document.add(new SortedSetDocValuesFacetField(FeatureIndexFields.F_UID.getFieldName(),
            entry.getUuid().toString()));
}