Java HLL类代码示例

import net.agkn.hll.HLL; //导入依赖的package包/类
private static void assertCardinality(final HLLType hllType, final Collection<Long> items)
       throws CloneNotSupportedException {
    for(int log2m=MINIMUM_LOG2M_PARAM; log2m<=16; log2m++) {
        for(int regw=MINIMUM_REGWIDTH_PARAM; regw<=MAXIMUM_REGWIDTH_PARAM; regw++) {
            for(int expthr=MINIMUM_EXPTHRESH_PARAM; expthr<=MAXIMUM_EXPTHRESH_PARAM; expthr++ ) {
                for(final boolean sparse: new boolean[]{true, false}) {
                    HLL hll = new HLL(log2m, regw, expthr, sparse, hllType);
                    for(final Long item: items) {
                        hll.addRaw(item);
                    }
                    HLL copy = HLL.fromBytes(hll.toBytes());
                    assertEquals(copy.cardinality(), hll.cardinality());
                    assertEquals(copy.getType(), hll.getType());
                    assertEquals(copy.toBytes(), hll.toBytes());

                    HLL clone = hll.clone();
                    assertEquals(clone.cardinality(), hll.cardinality());
                    assertEquals(clone.getType(), hll.getType());
                    assertEquals(clone.toBytes(), hll.toBytes());
                }
            }
        }
    }
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Cumulatively adds random values to an EMPTY HLL.
 *
 * Format: cumulative add
 * Tests:
 * - EMPTY, EXPLICIT, SPARSE, PROBABILSTIC addition
 * - EMPTY to EXPLICIT promotion
 * - EXPLICIT to SPARSE promotion
 * - SPARSE to FULL promotion
 */
private static void globalStepTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "comprehensive_promotion", TestType.ADD);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    initLineAdd(output, hll, schemaVersion);

    for(int i=0; i<10000/*arbitrary*/; i++) {
        cumulativeAddLine(output, hll, random.nextLong(), schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Cumulatively sets successive registers to:
 *
 *     <code>(registerIndex % REGISTER_MAX_VALUE) + 1</code>
 *
 * by adding specifically constructed values to a SPARSE HLL.
 * Does not induce promotion.
 *
 * Format: cumulative add
 * Tests:
 * - SPARSE addition (predictable)
 */
private static void sparseStepTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_step", TestType.ADD);

    // the accumulator, starts empty sparse probabilistic
    final HLL hll = newHLL(HLLType.SPARSE);
    initLineAdd(output, hll, schemaVersion);

    for(int i=0; i<SPARSE_THRESHOLD; i++) {
        final long rawValue = constructHLLValue(LOG2M, i, ((i % REGISTER_MAX_VALUE) + 1));
        cumulativeAddLine(output, hll, rawValue, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Cumulatively sets random registers of a SPARSE HLL to
 * random values by adding random values. Does not induce promotion.
 *
 * Format: cumulative add
 * Tests:
 * - SPARSE addition (random)
 */
private static void sparseRandomTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_random", TestType.ADD);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.SPARSE);
    initLineAdd(output, hll, schemaVersion);

    for(int i=0; i<SPARSE_THRESHOLD; i++) {
        final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
        final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
        final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);

        cumulativeAddLine(output, hll, rawValue, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Cumulatively sets the first register (index 0) to value 2, the last
 * register (index m-1) to value 2, and then sets registers with indices in
 * the range 2 to (sparseCutoff + 2) to value 1 to trigger promotion.
 *
 * This tests for register alignment in the promotion from SPARSE
 * to FULL.
 *
 * Format: cumulative add
 * Tests:
 * - SPARSE addition
 * - SPARSE to FULL promotion
 */
private static void sparseEdgeTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_edge", TestType.ADD);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.SPARSE);
    initLineAdd(output, hll, schemaVersion);

    final long firstValue = constructHLLValue(LOG2M, 0, 2);
    cumulativeAddLine(output, hll, firstValue, schemaVersion);

    final long lastValue = constructHLLValue(LOG2M, (1 << LOG2M) - 1, 2);
    cumulativeAddLine(output, hll, lastValue, schemaVersion);

    for(int i=2; i<(SPARSE_THRESHOLD + 2); i++) {
        final long middleValue = constructHLLValue(LOG2M, i, 1);

        cumulativeAddLine(output, hll, middleValue, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with EXPLICIT HLLs, each containing a
 * single random value.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U EXPLICIT
 * - EXPLICIT U EXPLICIT
 * - EXPLICIT to SPARSE promotion
 * - SPARSE U EXPLICIT
 */
private static void explicitPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "explicit_promotion", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<(EXPLICIT_THRESHOLD+500)/*should be greater than promotion cutoff*/; i++) {
        // make an EXPLICIT set and populate with cardinality 1
        final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
        explicitHLL.addRaw(random.nextLong());

        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with EXPLICIT HLLs, each having a single
 * random value, twice in a row to verify that the set properties are
 * satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U EXPLICIT
 * - EXPLICIT U EXPLICIT
 */
private static void explicitOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "explicit_explicit", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<EXPLICIT_THRESHOLD; i++) {
        // make an EXPLICIT set and populate with cardinality 1
        final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
        explicitHLL.addRaw(random.nextLong());

        // union it into the accumulator twice, to test overlap (cardinality should not change)
        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with SPARSE HLLs, each
 * having a single register set, twice in a row to verify that the set
 * properties are satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U SPARSE
 * - SPARSE U SPARSE
 */
private static void sparseProbabilisticOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_sparse", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<SPARSE_THRESHOLD; i++) {
        // make a SPARSE set and populate with cardinality 1
        final HLL sparseHLL = newHLL(HLLType.SPARSE);
        final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
        final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
        final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
        sparseHLL.addRaw(rawValue);

        cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with FULL HLLs, each having
 * many registers set, twice in a row to verify that the set properties are
 * satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U FULL
 * - FULL U FULL
 */
private static void probabilisticUnionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "probabilistic_probabilistic", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<1000/*number of rows to generate*/; i++) {
        // make a FULL set and populate with
        final HLL fullHLL = newHLL(HLLType.FULL);
        final int elementCount = random.nextInt(10000/*arbitrary maximum cardinality*/);
        for(int j=0;j<elementCount;j++) {
            fullHLL.addRaw(random.nextLong());
        }

        cumulativeUnionLine(output, hll, fullHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with random HLLs.
 *
 * Format: cumulative union
 * Tests:
 * - hopefully all union possibilities
 */
private static void globalUnionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "comprehensive", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<1000/*number of rows to generate*/; i++) {
        final HLL randomHLL = generateRandomHLL(random);
        cumulativeUnionLine(output, hll, randomHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
@Override
public void iterate(AggregationBuffer agg, Object[] parameters)
    throws HiveException {
  HLLBuffer hllbuf = (HLLBuffer) agg;
  switch(hll_mode){
  case HASHING:
    try {
      // TODO Need to test the performance between this and typed primitives
      for (int i=0; i<parameters.length; i++) {
        Writable writable = (Writable)primitiveOIs.get(i).getPrimitiveWritableObject(parameters[i]);
        if (writable!=null)
          writable.write(dos);
      }
      dos.flush();
      hllbuf.hll.addRaw(hashFunc.hashBytes(baos.toByteArray()).asLong());
      baos.reset();
    } catch (IOException e) {
      throw new HiveException(e);
    }
    break;
  case MERGING:
    LazyBinary lb = (LazyBinary) hllOI.getStructFieldData(
        parameters[0], 
        hllOI.getStructFieldRef(SIGNATURE));
    HLL other = HLL.fromBytes(lb.getWritableObject().copyBytes());
    hllbuf.hll.union(other);
  }
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
@Override
public void merge(AggregationBuffer agg, Object partial)
    throws HiveException {
  HLLBuffer hllbuf = (HLLBuffer)agg;
  if (partial!=null){
    BytesWritable bw = intermediateOI.getPrimitiveWritableObject(partial);
    HLL other = HLL.fromBytes(bw.copyBytes());
    hllbuf.hll.union(other);
  }
  
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Unions an EMPTY accumulator with SPARSE HLLs, each
 * having one register set.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U SPARSE
 * - SPARSE U SPARSE
 * - SPARSE promotion
 * - SPARSE U FULL
 */
private static void sparseProbabilisticPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_promotion", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);


    for(int i=0; i<(SPARSE_THRESHOLD + 1000)/*should be greater than promotion cutoff*/; i++) {
        // make a SPARSE set and populate with cardinality 1
        final HLL sparseHLL = newHLL(HLLType.SPARSE);

        final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
        final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
        final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
        sparseHLL.addRaw(rawValue);

        cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Returns the algorithm-specific cardinality of the specified {@link HLL}
 * as a {@link String} appropriate for comparison with the algorithm-specific
 * cardinality provided by the PostgreSQL implementation.
 *
 * @param  hll the HLL whose algorithm-specific cardinality is to be printed.
 *         This cannot be <code>null</code>.
 * @return the algorithm-specific cardinality of the instance as a PostgreSQL-
 *         compatible String. This will never be <code>null</code>
 */
private static String stringCardinality(final HLL hll) {
    switch(hll.getType()) {
        case EMPTY:
            return "0";
        case EXPLICIT:/*promotion has not yet occurred*/
            return Long.toString(hll.cardinality());
        case SPARSE:
            return Double.toString(hll.sparseProbabilisticAlgorithmCardinality());
        case FULL:
            return Double.toString(hll.fullProbabilisticAlgorithmCardinality());
        default:
            throw new RuntimeException("Unknown HLL type " + hll.getType());
    }
}
 

import net.agkn.hll.HLL; //导入依赖的package包/类
/**
 * Tests that {@link HLLUtil#largeEstimatorCutoff(int, int)} is the same
 * as a trivial implementation.
 */
@Test
public void largeEstimatorCutoffTest() {
    for(int log2m=HLL.MINIMUM_LOG2M_PARAM; log2m<=HLL.MAXIMUM_LOG2M_PARAM; log2m++) {
        for(int regWidth=HLL.MINIMUM_REGWIDTH_PARAM; regWidth<=HLL.MINIMUM_REGWIDTH_PARAM; regWidth++) {
            final double cutoff = HLLUtil.largeEstimatorCutoff(log2m, regWidth);

            // See blog post (http://research.neustar.biz/2013/01/24/hyperloglog-googles-take-on-engineering-hll/)
            // and original paper (Fig. 3) for information on 2^L and
            // "large range correction" cutoff.
            final double expected = Math.pow(2, Math.pow(2, regWidth) - 2 + log2m) / 30.0;
            assertEquals(cutoff, expected);
        }
    }
}