Java PackedInts.Reader方法代码示例

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private SortedDocValues loadBytesFixedSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
  CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_DAT,
                              Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START,
                              Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
  CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_IDX,
                               Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START,
                               Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);

  final int fixedLength = data.readInt();
  final int valueCount = index.readInt();

  PagedBytes bytes = new PagedBytes(16);
  bytes.copy(data, fixedLength * (long) valueCount);
  final PagedBytes.Reader bytesReader = bytes.freeze(true);
  final PackedInts.Reader reader = PackedInts.getReader(index);
  ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());

  return correctBuggyOrds(new SortedDocValues() {
    @Override
    public int getOrd(int docID) {
      return (int) reader.get(docID);
    }

    @Override
    public BytesRef lookupOrd(int ord) {
      final BytesRef term = new BytesRef();
      bytesReader.fillSlice(term, fixedLength * (long) ord, fixedLength);
      return term;
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  });
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private SortedDocValues loadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
  CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT,
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START,
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
  CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX,
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START,
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);

  long maxAddress = index.readLong();
  PagedBytes bytes = new PagedBytes(16);
  bytes.copy(data, maxAddress);
  final PagedBytes.Reader bytesReader = bytes.freeze(true);
  final PackedInts.Reader addressReader = PackedInts.getReader(index);
  final PackedInts.Reader ordsReader = PackedInts.getReader(index);

  final int valueCount = addressReader.size() - 1;
  ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());

  return correctBuggyOrds(new SortedDocValues() {
    @Override
    public int getOrd(int docID) {
      return (int)ordsReader.get(docID);
    }

    @Override
    public BytesRef lookupOrd(int ord) {
      final BytesRef term = new BytesRef();
      long startAddress = addressReader.get(ord);
      long endAddress = addressReader.get(ord+1);
      bytesReader.fillSlice(term, startAddress, (int)(endAddress - startAddress));
      return term;
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  });
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private SortedDocValues loadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
  CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, 
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, 
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
  CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, 
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, 
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);

  long maxAddress = index.readLong();
  PagedBytes bytes = new PagedBytes(16);
  bytes.copy(data, maxAddress);
  final PagedBytes.Reader bytesReader = bytes.freeze(true);
  final PackedInts.Reader addressReader = PackedInts.getReader(index);
  final PackedInts.Reader ordsReader = PackedInts.getReader(index);
  
  final int valueCount = addressReader.size() - 1;
  
  return correctBuggyOrds(new SortedDocValues() {
    @Override
    public int getOrd(int docID) {
      return (int)ordsReader.get(docID);
    }

    @Override
    public void lookupOrd(int ord, BytesRef result) {
      long startAddress = addressReader.get(ord);
      long endAddress = addressReader.get(ord+1);
      bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  });
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private SortedDocValues loadBytesFixedSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
  CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_DAT,
                              Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START,
                              Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
  CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_IDX,
                               Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START,
                               Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);

  final int fixedLength = data.readInt();
  final int valueCount = index.readInt();

  PagedBytes bytes = new PagedBytes(16);
  bytes.copy(data, fixedLength * (long) valueCount);
  final PagedBytes.Reader bytesReader = bytes.freeze(true);
  final PackedInts.Reader reader = PackedInts.getReader(index);
  ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());

  return correctBuggyOrds(new SortedDocValues() {
    @Override
    public int getOrd(int docID) {
      return (int) reader.get(docID);
    }

    @Override
    public void lookupOrd(int ord, BytesRef result) {
      bytesReader.fillSlice(result, fixedLength * (long) ord, fixedLength);
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  });
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private SortedDocValues loadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
  CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT,
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START,
                              Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
  CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX,
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START,
                               Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);

  long maxAddress = index.readLong();
  PagedBytes bytes = new PagedBytes(16);
  bytes.copy(data, maxAddress);
  final PagedBytes.Reader bytesReader = bytes.freeze(true);
  final PackedInts.Reader addressReader = PackedInts.getReader(index);
  final PackedInts.Reader ordsReader = PackedInts.getReader(index);

  final int valueCount = addressReader.size() - 1;
  ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addressReader.ramBytesUsed() + ordsReader.ramBytesUsed());

  return correctBuggyOrds(new SortedDocValues() {
    @Override
    public int getOrd(int docID) {
      return (int)ordsReader.get(docID);
    }

    @Override
    public void lookupOrd(int ord, BytesRef result) {
      long startAddress = addressReader.get(ord);
      long endAddress = addressReader.get(ord+1);
      bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
    }

    @Override
    public int getValueCount() {
      return valueCount;
    }
  });
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
/**
 * Return a {@link org.apache.lucene.util.packed.PackedInts.Reader} instance mapping every doc ID to its first ordinal + 1 if it exists and 0 otherwise.
 */
public PackedInts.Reader getFirstOrdinals() {
    return ordinals.firstOrdinals;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private BinaryDocValues loadBytesVarStraight(FieldInfo field) throws IOException {
  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "dat");
  String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "idx");
  IndexInput data = null;
  IndexInput index = null;
  boolean success = false;
  try {
    data = dir.openInput(dataName, state.context);
    CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT,
                                Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_START,
                                Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
    index = dir.openInput(indexName, state.context);
    CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX,
                                 Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_START,
                                 Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
    long totalBytes = index.readVLong();
    PagedBytes bytes = new PagedBytes(16);
    bytes.copy(data, totalBytes);
    final PagedBytes.Reader bytesReader = bytes.freeze(true);
    final PackedInts.Reader reader = PackedInts.getReader(index);
    CodecUtil.checkEOF(data);
    CodecUtil.checkEOF(index);
    success = true;
    ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
    return new BinaryDocValues() {
      @Override
      public BytesRef get(int docID) {
        final BytesRef term = new BytesRef();
        long startAddress = reader.get(docID);
        long endAddress = reader.get(docID+1);
        bytesReader.fillSlice(term, startAddress, (int)(endAddress - startAddress));
        return term;
      }
    };
  } finally {
    if (success) {
      IOUtils.close(data, index);
    } else {
      IOUtils.closeWhileHandlingException(data, index);
    }
  }
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private BinaryDocValues loadBytesFixedDeref(FieldInfo field) throws IOException {
  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "dat");
  String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "idx");
  IndexInput data = null;
  IndexInput index = null;
  boolean success = false;
  try {
    data = dir.openInput(dataName, state.context);
    CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT,
                                Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START,
                                Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
    index = dir.openInput(indexName, state.context);
    CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX,
                                 Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START,
                                 Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);

    final int fixedLength = data.readInt();
    final int valueCount = index.readInt();
    PagedBytes bytes = new PagedBytes(16);
    bytes.copy(data, fixedLength * (long) valueCount);
    final PagedBytes.Reader bytesReader = bytes.freeze(true);
    final PackedInts.Reader reader = PackedInts.getReader(index);
    CodecUtil.checkEOF(data);
    CodecUtil.checkEOF(index);
    ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
    success = true;
    return new BinaryDocValues() {
      @Override
      public BytesRef get(int docID) {
        final BytesRef term = new BytesRef();
        final long offset = fixedLength * reader.get(docID);
        bytesReader.fillSlice(term, offset, fixedLength);
        return term;
      }
    };
  } finally {
    if (success) {
      IOUtils.close(data, index);
    } else {
      IOUtils.closeWhileHandlingException(data, index);
    }
  }
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private BinaryDocValues loadBytesVarDeref(FieldInfo field) throws IOException {
  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "dat");
  String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "idx");
  IndexInput data = null;
  IndexInput index = null;
  boolean success = false;
  try {
    data = dir.openInput(dataName, state.context);
    CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT,
                                Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START,
                                Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
    index = dir.openInput(indexName, state.context);
    CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX,
                                 Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START,
                                 Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);

    final long totalBytes = index.readLong();
    final PagedBytes bytes = new PagedBytes(16);
    bytes.copy(data, totalBytes);
    final PagedBytes.Reader bytesReader = bytes.freeze(true);
    final PackedInts.Reader reader = PackedInts.getReader(index);
    CodecUtil.checkEOF(data);
    CodecUtil.checkEOF(index);
    ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + reader.ramBytesUsed());
    success = true;
    return new BinaryDocValues() {
      
      @Override
      public BytesRef get(int docID) {
        final BytesRef term = new BytesRef();
        long startAddress = reader.get(docID);
        BytesRef lengthBytes = new BytesRef();
        bytesReader.fillSlice(lengthBytes, startAddress, 1);
        byte code = lengthBytes.bytes[lengthBytes.offset];
        if ((code & 128) == 0) {
          // length is 1 byte
          bytesReader.fillSlice(term, startAddress + 1, (int) code);
        } else {
          bytesReader.fillSlice(lengthBytes, startAddress + 1, 1);
          int length = ((code & 0x7f) << 8) | (lengthBytes.bytes[lengthBytes.offset] & 0xff);
          bytesReader.fillSlice(term, startAddress + 2, length);
        }
        return term;
      }
    };
  } finally {
    if (success) {
      IOUtils.close(data, index);
    } else {
      IOUtils.closeWhileHandlingException(data, index);
    }
  }
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
  maxDoc = si.getDocCount();
  int[] docBases = new int[16];
  long[] startPointers = new long[16];
  int[] avgChunkDocs = new int[16];
  long[] avgChunkSizes = new long[16];
  PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];
  PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];

  final int packedIntsVersion = fieldsIndexIn.readVInt();

  int blockCount = 0;

  for (;;) {
    final int numChunks = fieldsIndexIn.readVInt();
    if (numChunks == 0) {
      break;
    }
    if (blockCount == docBases.length) {
      final int newSize = ArrayUtil.oversize(blockCount + 1, 8);
      docBases = Arrays.copyOf(docBases, newSize);
      startPointers = Arrays.copyOf(startPointers, newSize);
      avgChunkDocs = Arrays.copyOf(avgChunkDocs, newSize);
      avgChunkSizes = Arrays.copyOf(avgChunkSizes, newSize);
      docBasesDeltas = Arrays.copyOf(docBasesDeltas, newSize);
      startPointersDeltas = Arrays.copyOf(startPointersDeltas, newSize);
    }

    // doc bases
    docBases[blockCount] = fieldsIndexIn.readVInt();
    avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();
    final int bitsPerDocBase = fieldsIndexIn.readVInt();
    if (bitsPerDocBase > 32) {
      throw new CorruptIndexException("Corrupted bitsPerDocBase (resource=" + fieldsIndexIn + ")");
    }
    docBasesDeltas[blockCount] = PackedInts.getReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerDocBase);

    // start pointers
    startPointers[blockCount] = fieldsIndexIn.readVLong();
    avgChunkSizes[blockCount] = fieldsIndexIn.readVLong();
    final int bitsPerStartPointer = fieldsIndexIn.readVInt();
    if (bitsPerStartPointer > 64) {
      throw new CorruptIndexException("Corrupted bitsPerStartPointer (resource=" + fieldsIndexIn + ")");
    }
    startPointersDeltas[blockCount] = PackedInts.getReaderNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, bitsPerStartPointer);

    ++blockCount;
  }

  this.docBases = Arrays.copyOf(docBases, blockCount);
  this.startPointers = Arrays.copyOf(startPointers, blockCount);
  this.avgChunkDocs = Arrays.copyOf(avgChunkDocs, blockCount);
  this.avgChunkSizes = Arrays.copyOf(avgChunkSizes, blockCount);
  this.docBasesDeltas = Arrays.copyOf(docBasesDeltas, blockCount);
  this.startPointersDeltas = Arrays.copyOf(startPointersDeltas, blockCount);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
public IntsFromArray(PackedInts.Reader values, int minValue) {
  this.values = values;
  this.minValue = minValue;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
public LongsFromArray(PackedInts.Reader values, long minValue) {
  this.values = values;
  this.minValue = minValue;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
public SortedDocValuesImpl(PagedBytes.Reader bytes, PackedLongValues termOrdToBytesOffset, PackedInts.Reader docToTermOrd, int numOrd) {
  this.bytes = bytes;
  this.docToTermOrd = docToTermOrd;
  this.termOrdToBytesOffset = termOrdToBytesOffset;
  this.numOrd = numOrd;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
public BinaryDocValuesImpl(PagedBytes.Reader bytes, PackedInts.Reader docToOffset) {
  this.bytes = bytes;
  this.docToOffset = docToOffset;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入方法依赖的package包/类
private BinaryDocValues loadBytesVarStraight(FieldInfo field) throws IOException {
  String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "dat");
  String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name + "_" + Integer.toString(field.number), segmentSuffix, "idx");
  IndexInput data = null;
  IndexInput index = null;
  boolean success = false;
  try {
    data = dir.openInput(dataName, state.context);
    CodecUtil.checkHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, 
                                Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_START, 
                                Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
    index = dir.openInput(indexName, state.context);
    CodecUtil.checkHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, 
                                 Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_START, 
                                 Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
    long totalBytes = index.readVLong();
    PagedBytes bytes = new PagedBytes(16);
    bytes.copy(data, totalBytes);
    final PagedBytes.Reader bytesReader = bytes.freeze(true);
    final PackedInts.Reader reader = PackedInts.getReader(index);
    if (data.getFilePointer() != data.length()) {
      throw new CorruptIndexException("did not read all bytes from file \"" + dataName + "\": read " + data.getFilePointer() + " vs size " + data.length() + " (resource: " + data + ")");
    }
    if (index.getFilePointer() != index.length()) {
      throw new CorruptIndexException("did not read all bytes from file \"" + indexName + "\": read " + index.getFilePointer() + " vs size " + index.length() + " (resource: " + index + ")");
    }
    success = true;
    return new BinaryDocValues() {
      @Override
      public void get(int docID, BytesRef result) {
        long startAddress = reader.get(docID);
        long endAddress = reader.get(docID+1);
        bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
      }
    };
  } finally {
    if (success) {
      IOUtils.close(data, index);
    } else {
      IOUtils.closeWhileHandlingException(data, index);
    }
  }
}