Java PackedInts.DEFAULT属性代码示例

/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final Builder<Object> builder = new Builder<>(
      FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
      shareMaxTailLength, outputs, false, 
      PackedInts.DEFAULT, true, 15);
  
  BytesRefBuilder scratch = new BytesRefBuilder();
  BytesRef entry;
  final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.get().compareTo(entry) != 0) {
      builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : builder.finish();
}
 

public void testInternalFinalState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
  final boolean willRewrite = random().nextBoolean();
  final Builder<Long> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, willRewrite, PackedInts.DEFAULT, true, 15);
  builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRefBuilder()), outputs.getNoOutput());
  builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRefBuilder()), outputs.getNoOutput());
  final FST<Long> fst = builder.finish();
  StringWriter w = new StringWriter();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());

  // check for accept state at label t
  assertTrue(w.toString().indexOf("[label=\"t\" style=\"bold\"") != -1);
  // check for accept state at label n
  assertTrue(w.toString().indexOf("[label=\"n\" style=\"bold\"") != -1);
}
 

/**
 * Builds the final automaton from a list of entries.
 */
private FST<Object> buildAutomaton(BytesRefSorter sorter) throws IOException {
  // Build the automaton.
  final Outputs<Object> outputs = NoOutputs.getSingleton();
  final Object empty = outputs.getNoOutput();
  final Builder<Object> builder = new Builder<Object>(
      FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
      shareMaxTailLength, outputs, null, false, 
      PackedInts.DEFAULT, true, 15);
  
  BytesRef scratch = new BytesRef();
  BytesRef entry;
  final IntsRef scratchIntsRef = new IntsRef();
  int count = 0;
  BytesRefIterator iter = sorter.iterator();
  while((entry = iter.next()) != null) {
    count++;
    if (scratch.compareTo(entry) != 0) {
      builder.add(Util.toIntsRef(entry, scratchIntsRef), empty);
      scratch.copyBytes(entry);
    }
  }
  
  return count == 0 ? null : builder.finish();
}
 

public void testInternalFinalState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
  final boolean willRewrite = random().nextBoolean();
  final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, willRewrite, PackedInts.DEFAULT, true, 15);
  builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRef()), outputs.getNoOutput());
  builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRef()), outputs.getNoOutput());
  final FST<Long> fst = builder.finish();
  StringWriter w = new StringWriter();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());
  
  // check for accept state at label t
  assertTrue(w.toString().indexOf("[label=\"t\" style=\"bold\"") != -1);
  // check for accept state at label n
  assertTrue(w.toString().indexOf("[label=\"n\" style=\"bold\"") != -1);
}
 

/**
 * Creates a caching value source for the given underlying value source.
 * 
 * @param source
 *           the source of the uncached values.
 * @param maxDoc
 *           the maxDoc of the index.
 */
public FloatCachingValueSource(ValueSource source, int maxDoc, int cacheHint) {
   this.source = checkNotNull(source);
   checkArgument(maxDoc > 0, "Pre-condition violated: expression cacheHint >= 0 must be true.");
   checkArgument(cacheHint >= 0, "Pre-condition violated: expression cacheHint >= 0 must be true.");
   checkArgument(cacheHint <= 2, "Pre-condition violated: expression cacheHint <= 2 must be true.");

   if (CACHE_FAST == cacheHint) {
      // dead simple impl
      this.cache = new FloatArrayValueCache(maxDoc);
   } else if (CACHE_OFFSET_PACKED == cacheHint) {
      // mem efficient
      this.cache = new OffsetGrowableFloatWriter(
            OffsetGrowableFloatWriter.DEFAULT_PRECISION,
            4, maxDoc,
            PackedInts.DEFAULT);
   } else if (CACHE_SPARSE == cacheHint) {
      // mem efficient
      this.cache = new FloatArraySparseMutable(maxDoc, FloatArraySparseMutable.DEFAULT_PRECISION);
   } else {
      // TODO max memory efficient, but not suitable for larger datasets
      this.cache = null;
   }
}
 

/**
 * Builds an {@link Ordinals} instance from the builders current state.
 */
public Ordinals build() {
    final float acceptableOverheadRatio = PackedInts.DEFAULT;
    if (numMultiValuedDocs > 0 || MultiOrdinals.significantlySmallerThanSinglePackedOrdinals(maxDoc, numDocsWithValue, getValueCount(), acceptableOverheadRatio)) {
        // MultiOrdinals can be smaller than SinglePackedOrdinals for sparse fields
        return new MultiOrdinals(this, acceptableOverheadRatio);
    } else {
        return new SinglePackedOrdinals(this, acceptableOverheadRatio);
    }
}
 

public VisitTerms(String dirOut, String wordsFileIn, int inputMode, int prune, Outputs<T> outputs, boolean doPack, boolean noArcArrays) {
  this.dirOut = dirOut;
  this.wordsFileIn = wordsFileIn;
  this.inputMode = inputMode;
  this.outputs = outputs;
  this.doPack = doPack;

  builder = new Builder<>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, doPack, PackedInts.DEFAULT, !noArcArrays, 15);
}
 

public void testFinalOutputOnEndState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();

  final Builder<Long> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, random().nextBoolean(), PackedInts.DEFAULT, true, 15);
  builder.add(Util.toUTF32("stat", new IntsRefBuilder()), 17L);
  builder.add(Util.toUTF32("station", new IntsRefBuilder()), 10L);
  final FST<Long> fst = builder.finish();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
  StringWriter w = new StringWriter();
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());
  assertTrue(w.toString().indexOf("label=\"t/[7]\"") != -1);
}
 

public VisitTerms(String dirOut, String wordsFileIn, int inputMode, int prune, Outputs<T> outputs, boolean doPack, boolean noArcArrays) {
  this.dirOut = dirOut;
  this.wordsFileIn = wordsFileIn;
  this.inputMode = inputMode;
  this.outputs = outputs;
  this.doPack = doPack;

  builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null, doPack, PackedInts.DEFAULT, !noArcArrays, 15);
}
 

public void testFinalOutputOnEndState() throws Exception {
  final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);

  final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null, random().nextBoolean(), PackedInts.DEFAULT, true, 15);
  builder.add(Util.toUTF32("stat", new IntsRef()), 17L);
  builder.add(Util.toUTF32("station", new IntsRef()), 10L);
  final FST<Long> fst = builder.finish();
  //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
  StringWriter w = new StringWriter();
  Util.toDot(fst, w, false, false);
  w.close();
  //System.out.println(w.toString());
  assertTrue(w.toString().indexOf("label=\"t/[7]\"") != -1);
}
 

@Test
public void testFloatConversion() throws Exception {
   OffsetGrowableFloatWriter writer = new OffsetGrowableFloatWriter(OffsetGrowableFloatWriter.DEFAULT_PRECISION, 2, 4096, PackedInts.DEFAULT);

   for (int i = 0; i < 4096; i++) {
      float value = RandomUtils.nextFloat() * RandomUtils.nextFloat();
      writer.setFloat(i, value);
      assertEquals(value, writer.getFloat(i), OffsetGrowableFloatWriter.DEFAULT_PRECISION);
   }
}
 

@Test
public void testSize() throws Exception {
   OffsetGrowableFloatWriter writer = new OffsetGrowableFloatWriter(OffsetGrowableFloatWriter.DEFAULT_PRECISION, 2, 4096, PackedInts.DEFAULT);
   
   for (int i = 0; i < 4096; i++) {
      float value = RandomUtils.nextFloat() * RandomUtils.nextFloat();
      writer.setFloat(i, value);
      assertEquals(value, writer.getFloat(i), OffsetGrowableFloatWriter.DEFAULT_PRECISION);
   }
}
 

/**
 * Loads the segment information at segment load time.
 * 
 * @param indexEnum
 *          the term enum.
 * @param indexDivisor
 *          the index divisor.
 * @param tiiFileLength
 *          the size of the tii file, used to approximate the size of the
 *          buffer.
 * @param totalIndexInterval
 *          the total index interval.
 */
TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) throws IOException {
  this.totalIndexInterval = totalIndexInterval;
  indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor;
  skipInterval = indexEnum.skipInterval;
  // this is only an inital size, it will be GCed once the build is complete
  long initialSize = (long) (tiiFileLength * 1.5) / indexDivisor;
  PagedBytes dataPagedBytes = new PagedBytes(estimatePageBits(initialSize));
  PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput();

  final int bitEstimate = 1+MathUtil.log(tiiFileLength, 2);
  GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, PackedInts.DEFAULT);

  String currentField = null;
  List<String> fieldStrs = new ArrayList<>();
  int fieldCounter = -1;
  for (int i = 0; indexEnum.next(); i++) {
    Term term = indexEnum.term();
    if (currentField == null || !currentField.equals(term.field())) {
      currentField = term.field();
      fieldStrs.add(currentField);
      fieldCounter++;
    }
    TermInfo termInfo = indexEnum.termInfo();
    indexToTerms.set(i, dataOutput.getPosition());
    dataOutput.writeVInt(fieldCounter);
    dataOutput.writeString(term.text());
    dataOutput.writeVInt(termInfo.docFreq);
    if (termInfo.docFreq >= skipInterval) {
      dataOutput.writeVInt(termInfo.skipOffset);
    }
    dataOutput.writeVLong(termInfo.freqPointer);
    dataOutput.writeVLong(termInfo.proxPointer);
    dataOutput.writeVLong(indexEnum.indexPointer);
    for (int j = 1; j < indexDivisor; j++) {
      if (!indexEnum.next()) {
        break;
      }
    }
  }

  fields = new Term[fieldStrs.size()];
  for (int i = 0; i < fields.length; i++) {
    fields[i] = new Term(fieldStrs.get(i));
  }
  
  dataPagedBytes.freeze(true);
  dataInput = dataPagedBytes.getDataInput();
  indexToDataOffset = indexToTerms.getMutable();

  long ramBytesUsed = RamUsageEstimator.shallowSizeOf(fields);
  ramBytesUsed += RamUsageEstimator.shallowSizeOf(dataInput);
  ramBytesUsed += fields.length * RamUsageEstimator.shallowSizeOfInstance(Term.class);
  ramBytesUsed += dataPagedBytes.ramBytesUsed();
  ramBytesUsed += indexToDataOffset.ramBytesUsed();
  this.ramBytesUsed = ramBytesUsed;
}
 

public MemoryPostingsFormat() {
  this(false, PackedInts.DEFAULT);
}
 

/**
 * Loads the segment information at segment load time.
 * 
 * @param indexEnum
 *          the term enum.
 * @param indexDivisor
 *          the index divisor.
 * @param tiiFileLength
 *          the size of the tii file, used to approximate the size of the
 *          buffer.
 * @param totalIndexInterval
 *          the total index interval.
 */
TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) throws IOException {
  this.totalIndexInterval = totalIndexInterval;
  indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor;
  skipInterval = indexEnum.skipInterval;
  // this is only an inital size, it will be GCed once the build is complete
  long initialSize = (long) (tiiFileLength * 1.5) / indexDivisor;
  PagedBytes dataPagedBytes = new PagedBytes(estimatePageBits(initialSize));
  PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput();

  final int bitEstimate = 1+MathUtil.log(tiiFileLength, 2);
  GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, PackedInts.DEFAULT);

  String currentField = null;
  List<String> fieldStrs = new ArrayList<String>();
  int fieldCounter = -1;
  for (int i = 0; indexEnum.next(); i++) {
    Term term = indexEnum.term();
    if (currentField == null || !currentField.equals(term.field())) {
      currentField = term.field();
      fieldStrs.add(currentField);
      fieldCounter++;
    }
    TermInfo termInfo = indexEnum.termInfo();
    indexToTerms.set(i, dataOutput.getPosition());
    dataOutput.writeVInt(fieldCounter);
    dataOutput.writeString(term.text());
    dataOutput.writeVInt(termInfo.docFreq);
    if (termInfo.docFreq >= skipInterval) {
      dataOutput.writeVInt(termInfo.skipOffset);
    }
    dataOutput.writeVLong(termInfo.freqPointer);
    dataOutput.writeVLong(termInfo.proxPointer);
    dataOutput.writeVLong(indexEnum.indexPointer);
    for (int j = 1; j < indexDivisor; j++) {
      if (!indexEnum.next()) {
        break;
      }
    }
  }

  fields = new Term[fieldStrs.size()];
  for (int i = 0; i < fields.length; i++) {
    fields[i] = new Term(fieldStrs.get(i));
  }
  
  dataPagedBytes.freeze(true);
  dataInput = dataPagedBytes.getDataInput();
  indexToDataOffset = indexToTerms.getMutable();
}