Java IndexOptions.DOCS_ONLY属性代码示例

private void decodeMetaData() {
  if (!didDecode) {
    buffer.reset(current.output.bytes, current.output.offset, current.output.length);
    docFreq = buffer.readVInt();
    if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
      totalTermFreq = docFreq + buffer.readVLong();
    } else {
      totalTermFreq = -1;
    }
    postingsSpare.bytes = current.output.bytes;
    postingsSpare.offset = buffer.getPosition();
    postingsSpare.length = current.output.length - (buffer.getPosition() - current.output.offset);
    //System.out.println("  df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
    didDecode = true;
  }
}
 

@Override
public void startTerm() throws IOException {
  docIndex.mark();
  //System.out.println("SEPW: startTerm docIndex=" + docIndex);

  if (indexOptions != IndexOptions.DOCS_ONLY) {
    freqIndex.mark();
  }
  
  if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
    posIndex.mark();
    payloadStart = payloadOut.getFilePointer();
    lastPayloadLength = -1;
  }

  skipListWriter.resetSkip(docIndex, freqIndex, posIndex);
}
 

/** Adds a new doc in this term.  If this returns null
 *  then we just skip consuming positions/payloads. */
@Override
public void startDoc(int docID, int termDocFreq) throws IOException {

  final int delta = docID - lastDocID;
  //System.out.println("SEPW: startDoc: write doc=" + docID + " delta=" + delta + " out.fp=" + docOut);

  if (docID < 0 || (df > 0 && delta <= 0)) {
    throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " ) (docOut: " + docOut + ")");
  }

  if ((++df % skipInterval) == 0) {
    // TODO: -- awkward we have to make these two
    // separate calls to skipper
    //System.out.println("    buffer skip lastDocID=" + lastDocID);
    skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength);
    skipListWriter.bufferSkip(df);
  }

  lastDocID = docID;
  docOut.write(delta);
  if (indexOptions != IndexOptions.DOCS_ONLY) {
    //System.out.println("    sepw startDoc: write freq=" + termDocFreq);
    freqOut.write(termDocFreq);
  }
}
 

/** Optimized implementation. */
public int read(final int[] docs, final int[] freqs)
        throws IOException {
  final int length = docs.length;
  if (indexOptions == IndexOptions.DOCS_ONLY) {
    return readNoTf(docs, freqs, length);
  } else {
    int i = 0;
    while (i < length && count < df) {
      // manually inlined call to next() for speed
      final int docCode = freqStream.readVInt();
      doc += docCode >>> 1;       // shift off low bit
      if ((docCode & 1) != 0)       // if low bit is set
        freq = 1;         // freq is one
      else
        freq = freqStream.readVInt();     // else read freq
      count++;

      if (liveDocs == null || liveDocs.get(doc)) {
        docs[i] = doc;
        freqs[i] = freq;
        ++i;
      }
    }
    return i;
  }
}
 

@Override
public void close() throws IOException {
  if (out != null) {
    boolean success = false;
    try {
      // write field summary
      final long dirStart = out.getFilePointer();
      
      out.writeVInt(fields.size());
      for (FieldMetaData field : fields) {
        out.writeVInt(field.fieldInfo.number);
        out.writeVLong(field.numTerms);
        if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
          out.writeVLong(field.sumTotalTermFreq);
        }
        out.writeVLong(field.sumDocFreq);
        out.writeVInt(field.docCount);
        out.writeVInt(field.longsSize);
        field.dict.save(out);
      }
      writeTrailer(out, dirStart);
      CodecUtil.writeFooter(out);
      success = true;
    } finally {
      if (success) {
        IOUtils.close(out, postingsWriter);
      } else {
        IOUtils.closeWhileHandlingException(out, postingsWriter);
      }
      out = null;
    }
  }
}
 

@Override
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException {
  if (termCount > 0) {
    out.writeVInt(termCount);
    out.writeVInt(field.number);
    if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
      out.writeVLong(sumTotalTermFreq);
    }
    out.writeVLong(sumDocFreq);
    out.writeVInt(docCount);
    FST<BytesRef> fst = builder.finish();
    fst.save(out);
    //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
  }
}
 

public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) throws IOException {
  this.termCount = termCount;
  final int fieldNumber = in.readVInt();
  field = fieldInfos.fieldInfo(fieldNumber);
  if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) {
    sumTotalTermFreq = in.readVLong();
  } else {
    sumTotalTermFreq = -1;
  }
  sumDocFreq = in.readVLong();
  docCount = in.readVInt();
  
  fst = new FST<>(in, outputs);
}
 

private SepTermState setEmptyState() {
  SepTermState emptyState = new SepTermState();
  emptyState.docIndex = docOut.index();
  if (indexOptions != IndexOptions.DOCS_ONLY) {
    emptyState.freqIndex = freqOut.index();
    if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
      emptyState.posIndex = posOut.index();
    }
  }
  emptyState.payloadFP = 0;
  emptyState.skipFP = 0;
  return emptyState;
}
 

@Override
public void decodeTerm(long[] empty, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute) 
  throws IOException {
  final SepTermState termState = (SepTermState) _termState;
  termState.docIndex.read(in, absolute);
  if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
    termState.freqIndex.read(in, absolute);
    if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
      //System.out.println("  freqIndex=" + termState.freqIndex);
      termState.posIndex.read(in, absolute);
      //System.out.println("  posIndex=" + termState.posIndex);
      if (fieldInfo.hasPayloads()) {
        if (absolute) {
          termState.payloadFP = in.readVLong();
        } else {
          termState.payloadFP += in.readVLong();
        }
        //System.out.println("  payloadFP=" + termState.payloadFP);
      }
    }
  }

  if (termState.docFreq >= skipMinimum) {
    //System.out.println("   readSkip @ " + in.getPosition());
    if (absolute) {
      termState.skipFP = in.readVLong();
    } else {
      termState.skipFP += in.readVLong();
    }
    //System.out.println("  skipFP=" + termState.skipFP);
  } else if (absolute) {
    termState.skipFP = 0;
  }
}
 

SepDocsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) throws IOException {
  this.liveDocs = liveDocs;
  this.indexOptions = fieldInfo.getIndexOptions();
  omitTF = indexOptions == IndexOptions.DOCS_ONLY;
  storePayloads = fieldInfo.hasPayloads();

  // TODO: can't we only do this if consumer
  // skipped consuming the previous docs?
  docIndex.copyFrom(termState.docIndex);
  docIndex.seek(docReader);

  if (!omitTF) {
    freqIndex.copyFrom(termState.freqIndex);
    freqIndex.seek(freqReader);
  }

  docFreq = termState.docFreq;
  // NOTE: unused if docFreq < skipMinimum:
  skipFP = termState.skipFP;
  count = 0;
  doc = -1;
  accum = 0;
  freq = 1;
  skipped = false;

  return this;
}
 

@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
  IndexInput input = directory.openInput(fileName, iocontext);
  
  boolean success = false;
  try {
    CodecUtil.checkHeader(input, Lucene40FieldInfosFormat.CODEC_NAME, 
                                 Lucene40FieldInfosFormat.FORMAT_START, 
                                 Lucene40FieldInfosFormat.FORMAT_CURRENT);

    final int size = input.readVInt(); //read in the size
    FieldInfo infos[] = new FieldInfo[size];

    for (int i = 0; i < size; i++) {
      String name = input.readString();
      final int fieldNumber = input.readVInt();
      byte bits = input.readByte();
      boolean isIndexed = (bits & Lucene40FieldInfosFormat.IS_INDEXED) != 0;
      boolean storeTermVector = (bits & Lucene40FieldInfosFormat.STORE_TERMVECTOR) != 0;
      boolean omitNorms = (bits & Lucene40FieldInfosFormat.OMIT_NORMS) != 0;
      boolean storePayloads = (bits & Lucene40FieldInfosFormat.STORE_PAYLOADS) != 0;
      final IndexOptions indexOptions;
      if (!isIndexed) {
        indexOptions = null;
      } else if ((bits & Lucene40FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_ONLY;
      } else if ((bits & Lucene40FieldInfosFormat.OMIT_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS;
      } else if ((bits & Lucene40FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
      } else {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
      }

      // LUCENE-3027: past indices were able to write
      // storePayloads=true when omitTFAP is also true,
      // which is invalid.  We correct that, here:
      if (isIndexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
        storePayloads = false;
      }
      // DV Types are packed in one byte
      byte val = input.readByte();
      final LegacyDocValuesType oldValuesType = getDocValuesType((byte) (val & 0x0F));
      final LegacyDocValuesType oldNormsType = getDocValuesType((byte) ((val >>> 4) & 0x0F));
      final Map<String,String> attributes = input.readStringStringMap();;
      if (oldValuesType.mapping != null) {
        attributes.put(LEGACY_DV_TYPE_KEY, oldValuesType.name());
      }
      if (oldNormsType.mapping != null) {
        if (oldNormsType.mapping != DocValuesType.NUMERIC) {
          throw new CorruptIndexException("invalid norm type: " + oldNormsType + " (resource=" + input + ")");
        }
        attributes.put(LEGACY_NORM_TYPE_KEY, oldNormsType.name());
      }
      infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
        omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, -1, Collections.unmodifiableMap(attributes));
    }

    CodecUtil.checkEOF(input);
    FieldInfos fieldInfos = new FieldInfos(infos);
    success = true;
    return fieldInfos;
  } finally {
    if (success) {
      input.close();
    } else {
      IOUtils.closeWhileHandlingException(input);
    }
  }
}
 

@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
  ChecksumIndexInput input = directory.openChecksumInput(fileName, context);
  
  boolean success = false;
  try {
    int codecVersion = CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME, 
                                                    Lucene46FieldInfosFormat.FORMAT_START, 
                                                    Lucene46FieldInfosFormat.FORMAT_CURRENT);

    final int size = input.readVInt(); //read in the size
    FieldInfo infos[] = new FieldInfo[size];

    for (int i = 0; i < size; i++) {
      String name = input.readString();
      final int fieldNumber = input.readVInt();
      if (fieldNumber < 0) {
        throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber + " (resource=" + input + ")");
      }
      byte bits = input.readByte();
      boolean isIndexed = (bits & Lucene46FieldInfosFormat.IS_INDEXED) != 0;
      boolean storeTermVector = (bits & Lucene46FieldInfosFormat.STORE_TERMVECTOR) != 0;
      boolean omitNorms = (bits & Lucene46FieldInfosFormat.OMIT_NORMS) != 0;
      boolean storePayloads = (bits & Lucene46FieldInfosFormat.STORE_PAYLOADS) != 0;
      final IndexOptions indexOptions;
      if (!isIndexed) {
        indexOptions = null;
      } else if ((bits & Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_ONLY;
      } else if ((bits & Lucene46FieldInfosFormat.OMIT_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS;
      } else if ((bits & Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
      } else {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
      }

      // DV Types are packed in one byte
      byte val = input.readByte();
      final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
      final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
      final long dvGen = input.readLong();
      final Map<String,String> attributes = input.readStringStringMap();
      infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
        omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
    }
    
    if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
      CodecUtil.checkFooter(input);
    } else {
      CodecUtil.checkEOF(input);
    }
    FieldInfos fieldInfos = new FieldInfos(infos);
    success = true;
    return fieldInfos;
  } finally {
    if (success) {
      input.close();
    } else {
      IOUtils.closeWhileHandlingException(input);
    }
  }
}
 

@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
  IndexInput input = directory.openInput(fileName, iocontext);
  
  boolean success = false;
  try {
    final int format = input.readVInt();

    if (format > FORMAT_MINIMUM) {
      throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT);
    }
    if (format < FORMAT_CURRENT) {
      throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT);
    }

    final int size = input.readVInt(); //read in the size
    FieldInfo infos[] = new FieldInfo[size];

    for (int i = 0; i < size; i++) {
      String name = input.readString();
      final int fieldNumber = i;
      byte bits = input.readByte();
      boolean isIndexed = (bits & IS_INDEXED) != 0;
      boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
      boolean omitNorms = (bits & OMIT_NORMS) != 0;
      boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
      final IndexOptions indexOptions;
      if (!isIndexed) {
        indexOptions = null;
      } else if ((bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_ONLY;
      } else if ((bits & OMIT_POSITIONS) != 0) {
        if (format <= FORMAT_OMIT_POSITIONS) {
          indexOptions = IndexOptions.DOCS_AND_FREQS;
        } else {
          throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")");
        }
      } else {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
      }

      // LUCENE-3027: past indices were able to write
      // storePayloads=true when omitTFAP is also true,
      // which is invalid.  We correct that, here:
      if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
        storePayloads = false;
      }
      infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
        omitNorms, storePayloads, indexOptions, null, isIndexed && !omitNorms? DocValuesType.NUMERIC : null, -1, Collections.<String,String>emptyMap());
    }

    if (input.getFilePointer() != input.length()) {
      throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
    }
    FieldInfos fieldInfos = new FieldInfos(infos);
    success = true;
    return fieldInfos;
  } finally {
    if (success) {
      input.close();
    } else {
      IOUtils.closeWhileHandlingException(input);
    }
  }
}
 

@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
  IndexInput input = directory.openInput(fileName, iocontext);
  
  boolean success = false;
  try {
    CodecUtil.checkHeader(input, Lucene42FieldInfosFormat.CODEC_NAME, 
                                 Lucene42FieldInfosFormat.FORMAT_START, 
                                 Lucene42FieldInfosFormat.FORMAT_CURRENT);

    final int size = input.readVInt(); //read in the size
    FieldInfo infos[] = new FieldInfo[size];

    for (int i = 0; i < size; i++) {
      String name = input.readString();
      final int fieldNumber = input.readVInt();
      byte bits = input.readByte();
      boolean isIndexed = (bits & Lucene42FieldInfosFormat.IS_INDEXED) != 0;
      boolean storeTermVector = (bits & Lucene42FieldInfosFormat.STORE_TERMVECTOR) != 0;
      boolean omitNorms = (bits & Lucene42FieldInfosFormat.OMIT_NORMS) != 0;
      boolean storePayloads = (bits & Lucene42FieldInfosFormat.STORE_PAYLOADS) != 0;
      final IndexOptions indexOptions;
      if (!isIndexed) {
        indexOptions = null;
      } else if ((bits & Lucene42FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_ONLY;
      } else if ((bits & Lucene42FieldInfosFormat.OMIT_POSITIONS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS;
      } else if ((bits & Lucene42FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0) {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
      } else {
        indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
      }

      // DV Types are packed in one byte
      byte val = input.readByte();
      final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
      final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
      final Map<String,String> attributes = input.readStringStringMap();
      infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
        omitNorms, storePayloads, indexOptions, docValuesType, normsType, -1, Collections.unmodifiableMap(attributes));
    }

    CodecUtil.checkEOF(input);
    FieldInfos fieldInfos = new FieldInfos(infos);
    success = true;
    return fieldInfos;
  } finally {
    if (success) {
      input.close();
    } else {
      IOUtils.closeWhileHandlingException(input);
    }
  }
}
 

/**
 * Constructs a new FieldInfos from an array of FieldInfo objects
 */
public FieldInfos(FieldInfo[] infos) {
  boolean hasVectors = false;
  boolean hasProx = false;
  boolean hasPayloads = false;
  boolean hasOffsets = false;
  boolean hasFreq = false;
  boolean hasNorms = false;
  boolean hasDocValues = false;
  
  for (FieldInfo info : infos) {
    if (info.number < 0) {
      throw new IllegalArgumentException("illegal field number: " + info.number + " for field " + info.name);
    }
    FieldInfo previous = byNumber.put(info.number, info);
    if (previous != null) {
      throw new IllegalArgumentException("duplicate field numbers: " + previous.name + " and " + info.name + " have: " + info.number);
    }
    previous = byName.put(info.name, info);
    if (previous != null) {
      throw new IllegalArgumentException("duplicate field names: " + previous.number + " and " + info.number + " have: " + info.name);
    }
    
    hasVectors |= info.hasVectors();
    hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
    hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS_ONLY;
    hasOffsets |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    hasNorms |= info.hasNorms();
    hasDocValues |= info.hasDocValues();
    hasPayloads |= info.hasPayloads();
  }
  
  this.hasVectors = hasVectors;
  this.hasProx = hasProx;
  this.hasPayloads = hasPayloads;
  this.hasOffsets = hasOffsets;
  this.hasFreq = hasFreq;
  this.hasNorms = hasNorms;
  this.hasDocValues = hasDocValues;
  this.values = Collections.unmodifiableCollection(byNumber.values());
}