Java PackedInts类代码示例

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
@Override
public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
    finishLeaf();

    context = ctx;
    docDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
    buckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);

    return new LeafBucketCollector() {
        int lastDoc = 0;

        @Override
        public void collect(int doc, long bucket) throws IOException {
            docDeltas.add(doc - lastDoc);
            buckets.add(bucket);
            lastDoc = doc;
            maxBucket = Math.max(maxBucket, bucket);
        }
    };
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
private void addAddresses(FieldInfo field, Iterable<Number> values) throws IOException {
  meta.writeVInt(field.number);
  meta.writeByte(Lucene49DocValuesFormat.NUMERIC);
  meta.writeVInt(MONOTONIC_COMPRESSED);
  meta.writeLong(-1L);
  meta.writeLong(data.getFilePointer());
  meta.writeVLong(maxDoc);
  meta.writeVInt(PackedInts.VERSION_CURRENT);
  meta.writeVInt(BLOCK_SIZE);

  final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
  long addr = 0;
  writer.add(addr);
  for (Number v : values) {
    addr += v.longValue();
    writer.add(addr);
  }
  writer.finish();
  meta.writeLong(data.getFilePointer());
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
private void addAddresses(FieldInfo field, Iterable<Number> values) throws IOException {
  meta.writeVInt(field.number);
  meta.writeByte(Lucene410DocValuesFormat.NUMERIC);
  meta.writeVInt(MONOTONIC_COMPRESSED);
  meta.writeLong(-1L);
  meta.writeLong(data.getFilePointer());
  meta.writeVLong(maxDoc);
  meta.writeVInt(PackedInts.VERSION_CURRENT);
  meta.writeVInt(BLOCK_SIZE);

  final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
  long addr = 0;
  writer.add(addr);
  for (Number v : values) {
    addr += v.longValue();
    writer.add(addr);
  }
  writer.finish();
  meta.writeLong(data.getFilePointer());
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
void pforDecompress(byte token) {
  final int bitsPerValue = token & 0x1F;
  if (bitsPerValue == 0) {
    Arrays.fill(nextDocs, 0);
  } else {
    DECODERS[bitsPerValue].decode(data, offset, nextDocs, 0, ITERATIONS[bitsPerValue]);
    offset += BYTE_BLOCK_COUNTS[bitsPerValue];
  }
  if ((token & HAS_EXCEPTIONS) != 0) {
    // there are exceptions
    final int numExceptions = data[offset++];
    final int bitsPerException = data[offset++];
    final int numIterations = (numExceptions + DECODERS[bitsPerException].byteValueCount() - 1) / DECODERS[bitsPerException].byteValueCount();
    DECODERS[bitsPerException].decode(data, offset, nextExceptions, 0, numIterations);
    offset += PackedInts.Format.PACKED.byteCount(PackedInts.VERSION_CURRENT, numExceptions, bitsPerException);
    for (int i = 0; i < numExceptions; ++i) {
      nextDocs[data[offset++]] |= nextExceptions[i] << bitsPerValue;
    }
  }
  for (int previousDoc = docID, i = 0; i < BLOCK_SIZE; ++i) {
    final int doc = previousDoc + 1 + nextDocs[i];
    previousDoc = nextDocs[i] = doc;
  }
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
private long getPageMemoryUsage(PackedLongValues values, float acceptableOverheadRatio, int pageSize, long pageMinOrdinal, long pageMaxOrdinal) {
    int bitsRequired;
    long pageMemorySize = 0;
    PackedInts.FormatAndBits formatAndBits;
    if (pageMaxOrdinal == Long.MIN_VALUE) {
        // empty page - will use the null reader which just stores size
        pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT);

    } else {
        long pageMinValue = values.get(pageMinOrdinal);
        long pageMaxValue = values.get(pageMaxOrdinal);
        long pageDelta = pageMaxValue - pageMinValue;
        if (pageDelta != 0) {
            bitsRequired = pageDelta < 0 ? 64 : PackedInts.bitsRequired(pageDelta);
            formatAndBits = PackedInts.fastestFormatAndBits(pageSize, bitsRequired, acceptableOverheadRatio);
            pageMemorySize += formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, pageSize, formatAndBits.bitsPerValue) * RamUsageEstimator.NUM_BYTES_LONG;
            pageMemorySize += RamUsageEstimator.NUM_BYTES_LONG; // min value per page storage
        } else {
            // empty page
            pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT);
        }
    }
    return pageMemorySize;
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final Builder<Object> builder = new Builder<>(
      FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
      shareMaxTailLength, outputs, false, 
      PackedInts.DEFAULT, true, 15);
  
  BytesRefBuilder scratch = new BytesRefBuilder();
  BytesRef entry;
  final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.get().compareTo(entry) != 0) {
      builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : builder.finish();
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
private PackedLongValues getDeletes(List<AtomicReader> readers) {
  PackedLongValues.Builder deletes = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
  int deleteCount = 0;
  for (AtomicReader reader : readers) {
    final int maxDoc = reader.maxDoc();
    final Bits liveDocs = reader.getLiveDocs();
    for (int i = 0; i < maxDoc; ++i) {
      if (liveDocs != null && !liveDocs.get(i)) {
        ++deleteCount;
      } else {
        deletes.add(deleteCount);
      }
    }
  }
  return deletes.build();
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
public void testDateCompression() throws IOException {
  final Directory dir = new RAMDirectory();
  final IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
  final IndexWriter iwriter = new IndexWriter(dir, iwc);

  final long base = 13; // prime
  final long day = 1000L * 60 * 60 * 24;

  final Document doc = new Document();
  final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);
  doc.add(dvf);
  for (int i = 0; i < 300; ++i) {
    dvf.setLongValue(base + random().nextInt(1000) * day);
    iwriter.addDocument(doc);
  }
  iwriter.forceMerge(1);
  final long size1 = dirSize(dir);
  for (int i = 0; i < 50; ++i) {
    dvf.setLongValue(base + random().nextInt(1000) * day);
    iwriter.addDocument(doc);
  }
  iwriter.forceMerge(1);
  final long size2 = dirSize(dir);
  // make sure the new longs costed less than if they had only been packed
  assertTrue(size2 < size1 + (PackedInts.bitsRequired(day) * 50) / 8);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
public void testInternalFinalState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
  final boolean willRewrite = random().nextBoolean();
  final Builder<Long> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, willRewrite, PackedInts.DEFAULT, true, 15);
  builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRefBuilder()), outputs.getNoOutput());
  builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRefBuilder()), outputs.getNoOutput());
  final FST<Long> fst = builder.finish();
  StringWriter w = new StringWriter();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());

  // check for accept state at label t
  assertTrue(w.toString().indexOf("[label=\"t\" style=\"bold\"") != -1);
  // check for accept state at label n
  assertTrue(w.toString().indexOf("[label=\"n\" style=\"bold\"") != -1);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
TermsWriter(FieldInfo fieldInfo) {
  this.fieldInfo = fieldInfo;

  noOutputs = NoOutputs.getSingleton();

  // This Builder is just used transiently to fragment
  // terms into "good" blocks; we don't save the
  // resulting FST:
  blockBuilder = new Builder<Object>(FST.INPUT_TYPE.BYTE1,
                                     0, 0, true,
                                     true, Integer.MAX_VALUE,
                                     noOutputs,
                                     new FindBlocks(), false,
                                     PackedInts.COMPACT,
                                     true, 15);

  postingsWriter.setField(fieldInfo);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
public void testInternalFinalState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
  final boolean willRewrite = random().nextBoolean();
  final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, willRewrite, PackedInts.DEFAULT, true, 15);
  builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRef()), outputs.getNoOutput());
  builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRef()), outputs.getNoOutput());
  final FST<Long> fst = builder.finish();
  StringWriter w = new StringWriter();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());
  
  // check for accept state at label t
  assertTrue(w.toString().indexOf("[label=\"t\" style=\"bold\"") != -1);
  // check for accept state at label n
  assertTrue(w.toString().indexOf("[label=\"n\" style=\"bold\"") != -1);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
@Test
public void testWritePerformance() throws Exception {
   System.out.println("Writing " + NumberFormat.getInstance().format(MAX_DOCS) + " values.");
   System.out.println("Float array bytes: " + NumberFormat.getInstance().format(RamUsageEstimator.sizeOf(new float[MAX_DOCS])));
   OffsetGrowableFloatWriter writer = new OffsetGrowableFloatWriter(OffsetGrowableFloatWriter.DEFAULT_PRECISION, 2, MAX_DOCS, PackedInts.DEFAULT);
   long start = System.currentTimeMillis();
   
   for (int i = 0; i < MAX_DOCS; i++) {
      float value = RandomUtils.nextFloat() * RandomUtils.nextFloat();
      int j = RandomUtils.nextInt(MAX_DOCS);
      
      writer.setFloat(j, value);
      assertEquals(value, writer.getFloat(j), OffsetGrowableFloatWriter.DEFAULT_PRECISION);
   }
   
   System.out.println("OffsetGrowableFloatWriter bytes: " + NumberFormat.getInstance().format(writer.ramBytesUsed()) + " in " + (System.currentTimeMillis() - start) + "ms ...");
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
  meta.writeVInt(field.number);
  meta.writeByte(DiskDocValuesFormat.SORTED_SET);
  // write the ord -> byte[] as a binary field
  addBinaryField(field, values);
  // write the stream of ords as a numeric field
  // NOTE: we could return an iterator that delta-encodes these within a doc
  addNumericField(field, ords);
  
  // write the doc -> ord count as a absolute index to the stream
  meta.writeVInt(field.number);
  meta.writeByte(DiskDocValuesFormat.NUMERIC);
  meta.writeVInt(PackedInts.VERSION_CURRENT);
  meta.writeLong(data.getFilePointer());
  meta.writeVLong(maxDoc);
  meta.writeVInt(BLOCK_SIZE);

  final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
  long addr = 0;
  for (Number v : docToOrdCount) {
    addr += v.longValue();
    writer.add(addr);
  }
  writer.finish();
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
TermsWriter(FieldInfo fieldInfo) {
  this.fieldInfo = fieldInfo;

  noOutputs = NoOutputs.getSingleton();

  // This Builder is just used transiently to fragment
  // terms into "good" blocks; we don't save the
  // resulting FST:
  blockBuilder = new Builder<Object>(FST.INPUT_TYPE.BYTE1,
                                     0, 0, true,
                                     true, Integer.MAX_VALUE,
                                     noOutputs,
                                     new FindBlocks(), false,
                                     PackedInts.COMPACT,
                                     true, 15);

  this.longsSize = postingsWriter.setField(fieldInfo);
}
 

import org.apache.lucene.util.packed.PackedInts; //导入依赖的package包/类
/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final Builder<Object> builder = new Builder<Object>(
      FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
      shareMaxTailLength, outputs, null, false, 
      PackedInts.DEFAULT, true, 15);
  
  BytesRef scratch = new BytesRef();
  BytesRef entry;
  final IntsRef scratchIntsRef = new IntsRef();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.compareTo(entry) != 0) {
      builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : builder.finish();
}