Java HLL.addRaw方法代码示例

import net.agkn.hll.HLL; //导入方法依赖的package包/类
private static void assertCardinality(final HLLType hllType, final Collection<Long> items)
       throws CloneNotSupportedException {
    for(int log2m=MINIMUM_LOG2M_PARAM; log2m<=16; log2m++) {
        for(int regw=MINIMUM_REGWIDTH_PARAM; regw<=MAXIMUM_REGWIDTH_PARAM; regw++) {
            for(int expthr=MINIMUM_EXPTHRESH_PARAM; expthr<=MAXIMUM_EXPTHRESH_PARAM; expthr++ ) {
                for(final boolean sparse: new boolean[]{true, false}) {
                    HLL hll = new HLL(log2m, regw, expthr, sparse, hllType);
                    for(final Long item: items) {
                        hll.addRaw(item);
                    }
                    HLL copy = HLL.fromBytes(hll.toBytes());
                    assertEquals(copy.cardinality(), hll.cardinality());
                    assertEquals(copy.getType(), hll.getType());
                    assertEquals(copy.toBytes(), hll.toBytes());

                    HLL clone = hll.clone();
                    assertEquals(clone.cardinality(), hll.cardinality());
                    assertEquals(clone.getType(), hll.getType());
                    assertEquals(clone.toBytes(), hll.toBytes());
                }
            }
        }
    }
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Unions an EMPTY accumulator with EXPLICIT HLLs, each containing a
 * single random value.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U EXPLICIT
 * - EXPLICIT U EXPLICIT
 * - EXPLICIT to SPARSE promotion
 * - SPARSE U EXPLICIT
 */
private static void explicitPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "explicit_promotion", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<(EXPLICIT_THRESHOLD+500)/*should be greater than promotion cutoff*/; i++) {
        // make an EXPLICIT set and populate with cardinality 1
        final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
        explicitHLL.addRaw(random.nextLong());

        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Unions an EMPTY accumulator with EXPLICIT HLLs, each having a single
 * random value, twice in a row to verify that the set properties are
 * satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U EXPLICIT
 * - EXPLICIT U EXPLICIT
 */
private static void explicitOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "explicit_explicit", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<EXPLICIT_THRESHOLD; i++) {
        // make an EXPLICIT set and populate with cardinality 1
        final HLL explicitHLL = newHLL(HLLType.EXPLICIT);
        explicitHLL.addRaw(random.nextLong());

        // union it into the accumulator twice, to test overlap (cardinality should not change)
        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
        cumulativeUnionLine(output, hll, explicitHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Unions an EMPTY accumulator with SPARSE HLLs, each
 * having a single register set, twice in a row to verify that the set
 * properties are satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U SPARSE
 * - SPARSE U SPARSE
 */
private static void sparseProbabilisticOverlapTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_sparse", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<SPARSE_THRESHOLD; i++) {
        // make a SPARSE set and populate with cardinality 1
        final HLL sparseHLL = newHLL(HLLType.SPARSE);
        final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
        final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
        final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
        sparseHLL.addRaw(rawValue);

        cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Unions an EMPTY accumulator with FULL HLLs, each having
 * many registers set, twice in a row to verify that the set properties are
 * satisfied.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U FULL
 * - FULL U FULL
 */
private static void probabilisticUnionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "probabilistic_probabilistic", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);

    for(int i=0; i<1000/*number of rows to generate*/; i++) {
        // make a FULL set and populate with
        final HLL fullHLL = newHLL(HLLType.FULL);
        final int elementCount = random.nextInt(10000/*arbitrary maximum cardinality*/);
        for(int j=0;j<elementCount;j++) {
            fullHLL.addRaw(random.nextLong());
        }

        cumulativeUnionLine(output, hll, fullHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Unions an EMPTY accumulator with SPARSE HLLs, each
 * having one register set.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U SPARSE
 * - SPARSE U SPARSE
 * - SPARSE promotion
 * - SPARSE U FULL
 */
private static void sparseProbabilisticPromotionTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_promotion", TestType.UNION);

    final Random random = new Random(SEED);

    // the accumulator, starts empty
    final HLL hll = newHLL(HLLType.EMPTY);
    final HLL emptyHLL = newHLL(HLLType.EMPTY);
    cumulativeUnionLine(output, hll, emptyHLL, schemaVersion);


    for(int i=0; i<(SPARSE_THRESHOLD + 1000)/*should be greater than promotion cutoff*/; i++) {
        // make a SPARSE set and populate with cardinality 1
        final HLL sparseHLL = newHLL(HLLType.SPARSE);

        final int registerIndex = Math.abs(random.nextInt()) % REGISTER_COUNT;
        final int registerValue = ((Math.abs(random.nextInt()) % REGISTER_MAX_VALUE) + 1);
        final long rawValue = constructHLLValue(LOG2M, registerIndex, registerValue);
        sparseHLL.addRaw(rawValue);

        cumulativeUnionLine(output, hll, sparseHLL, schemaVersion);
    }

    output.flush();
    output.close();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
public static void main(String... args) {

HashFunction hasher = Hashing.murmur3_128();

final Integer[] data = new Integer[]{1, 1, 2, 2, 3, 3, 4, 4, 5, 5};

final HLL hll = new HLL(13/*log2m*/, 5/*registerWidth*/);

for (int item : data) {
  final long hashedValue = hasher.newHasher().putInt(item).hash().asLong();
  hll.addRaw(hashedValue);
}

System.out.println("Distinct count = " + hll.cardinality());

}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Cumulatively unions "underpopulated" FULL HLLs into the
 * accumulator to verify the correct behavior from the PostgreSQL implementation.
 * The PostgreSQL implementation's representations of probabilistic HLLs should
 * depend exclusively on the chosen SPARSE-to-FULL cutoff.
 *
 * Format: cumulative union
 * Tests:
 * - EMPTY U "underpopulated" FULL => SPARSE
 * - SPARSE U "underpopulated" FULL => SPARSE
 * - SPARSE U "barely underpopulated" FULL => FULL
 */
private static void sparseFullRepresentationTest(final ISchemaVersion schemaVersion) throws IOException {
    final FileWriter output = openOutput(schemaVersion, "sparse_full_representation", TestType.UNION);

    final HLL emptyHLL1 = newHLL(HLLType.EMPTY);
    final HLL emptyHLL2 = newHLL(HLLType.EMPTY);

    cumulativeUnionLine(output, emptyHLL1, emptyHLL2, schemaVersion);

    // NOTE:  In this test the sparseReference will be the "expected" value
    //        from the C representation, since it doesn't choose representation
    //        based on original encoding, but rather on the promotion rules
    //        and the declared type of the "receiving" field.
    //        It is the manually-constructed union result.

    // "underpopulated" FULL U EMPTY => SPARSE
    final HLL fullHLL = newHLL(HLLType.FULL);
    fullHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));

    final HLL sparseHLL = newHLL(HLLType.SPARSE);
    sparseHLL.addRaw(constructHLLValue(LOG2M, 0/*ix*/, 1/*val*/));

    output.write(stringCardinality(fullHLL) + "," + toByteA(fullHLL, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
    output.flush();

    // "underpopulated" FULL (small) U SPARSE (small) => SPARSE
    final HLL fullHLL2 = newHLL(HLLType.FULL);
    fullHLL2.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));

    sparseHLL.addRaw(constructHLLValue(LOG2M, 1/*ix*/, 1/*val*/));

    output.write(stringCardinality(fullHLL2) + "," + toByteA(fullHLL2, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
    output.flush();

    // "underpopulated" FULL (just on edge) U SPARSE (small) => FULL
    final HLL fullHLL3 = newHLL(HLLType.FULL);
    for(int i=2; i<(SPARSE_THRESHOLD + 1); i++) {
        fullHLL3.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
        sparseHLL.addRaw(constructHLLValue(LOG2M, i/*ix*/, 1/*val*/));
    }

    output.write(stringCardinality(fullHLL3) + "," + toByteA(fullHLL3, schemaVersion) + "," + stringCardinality(sparseHLL) + "," + toByteA(sparseHLL, schemaVersion) + "\n");
    output.flush();
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Generates a random HLL and populates it with random values.
 *
 * @param  random the {@link Random random number generator} used to populate
 *         the HLL. This cannot be <code>null</code>.
 * @return the populated HLL. This will never be <code>null</code>.
 */
public static HLL generateRandomHLL(final Random random) {
    final int randomTypeInt = random.nextInt(HLLType.values().length);
    final HLLType type;
    switch(randomTypeInt) {
        case 0:
            type = HLLType.EMPTY;
            break;
        case 1:
            type = HLLType.EXPLICIT;
            break;
        case 2:
            type = HLLType.FULL;
            break;
        case 3:
            type = HLLType.EMPTY;
            break;
        case 4:
            type = HLLType.SPARSE;
            break;
        default:
            throw new RuntimeException("Unassigned type int " + randomTypeInt);
    }

    final int cardinalityCap;
    final int cardinalityBaseline;

    switch(type) {
        case EMPTY:
            return newHLL(HLLType.EMPTY);
        case EXPLICIT:
            cardinalityCap = EXPLICIT_THRESHOLD;
            cardinalityBaseline = 1;
            break;
        case SPARSE:
            cardinalityCap = SPARSE_THRESHOLD;
            cardinalityBaseline = (EXPLICIT_THRESHOLD + 1);
            break;
        case FULL:
            cardinalityCap = 100000;
            cardinalityBaseline = (SPARSE_THRESHOLD*10);
            break;
        default:
            throw new RuntimeException("We should never be here.");
    }

    final HLL hll = newHLL(HLLType.EMPTY);
    for(int i=0; i<cardinalityBaseline; i++) {
        hll.addRaw(random.nextLong());
    }
    for(int i=0; i<random.nextInt(cardinalityCap - cardinalityBaseline); i++) {
        hll.addRaw(random.nextLong());
    }

    return hll;
}
 

import net.agkn.hll.HLL; //导入方法依赖的package包/类
/**
 * Writes out a {@link TestType#ADD}-formatted test line.
 *
 * @param  output The output {@link FileWriter writer}. This cannot be <code>null</code>.
 * @param  hll The "accumulator" HLL instance. This cannot be <code>null</code>.
 * @param  rawValue The raw value added to the HLL.
 * @param  schemaVersion the schema with which to serialize the HLLs. This cannot
 *         be <code>null</code>.
 */
private static void cumulativeAddLine(final FileWriter output, final HLL hll, final long rawValue, final ISchemaVersion schemaVersion) throws IOException {
    hll.addRaw(rawValue);
    final String accumulatorCardinality = stringCardinality(hll);

    output.write(accumulatorCardinality + "," + rawValue + "," + toByteA(hll, schemaVersion) + "\n");
    output.flush();
}