Java HyperLogLogPlus类代码示例

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public Iterable<Element> _apply(final String line) {
    final List<Element> elements = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        final HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(8, 8);
        hyperLogLogPlus.offer("B" + i);

        final Entity entity = new Entity.Builder()
                .group("cardinality")
                .vertex("A")
                .property("approxCardinality", hyperLogLogPlus)
                .build();
        elements.add(entity);
    }
    return elements;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
public StringQuality(long countValid, long countInvalid, long countMissing, long countTotal, long countEmptyString,
                long countAlphabetic, long countNumerical, long countWordCharacter, long countWhitespace,
                double relativeSD) {
    /*
     * The algorithm used is based on streamlib's implementation of "HyperLogLog in Practice:
     * Algorithmic Engineering of a State of The Art Cardinality Estimation Algorithm", available
     * <a href="http://dx.doi.org/10.1145/2452376.2452456">here</a>.
     *
     * The relative accuracy is approximately `1.054 / sqrt(2^p)`. Setting
     * a nonzero `sp > p` in HyperLogLogPlus(p, sp) would trigger sparse
     * representation of registers, which may reduce the memory consumption
     * and increase accuracy when the cardinality is small.
     */
    this(countValid, countInvalid, countMissing, countTotal, countEmptyString, countAlphabetic, countNumerical,
                    countWordCharacter, countWhitespace,
                    new HyperLogLogPlus((int) Math.ceil(2.0 * Math.log(1.054 / relativeSD) / Math.log(2)), 0));
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public void registerClasses(final Kryo kryo) {
    kryo.register(Entity.class, new EntityKryoSerializer());
    kryo.register(Edge.class, new EdgeKryoSerializer());
    kryo.register(Properties.class);
    kryo.register(FreqMap.class, new FreqMapKryoSerializer());
    kryo.register(HyperLogLogPlus.class, new HyperLogLogPlusKryoSerializer());
    kryo.register(HllSketch.class, new HllSketchKryoSerializer());
    kryo.register(Union.class, new HllUnionKryoSerializer());
    kryo.register(LongsSketch.class, new LongsSketchKryoSerializer());
    kryo.register(ItemsSketch.class, new StringsSketchKryoSerializer());
    kryo.register(com.yahoo.sketches.quantiles.ItemsSketch.class, new uk.gov.gchq.gaffer.spark.serialisation.kryo.impl.datasketches.quantiles.StringsSketchKryoSerializer());
    kryo.register(ItemsUnion.class, new StringsUnionKryoSerializer());
    kryo.register(ReservoirLongsSketch.class, new ReservoirLongsSketchKryoSerializer());
    kryo.register(ReservoirLongsUnion.class, new ReservoirLongsUnionKryoSerializer());
    kryo.register(TypeValue.class, new TypeValueKryoSerializer());
    kryo.register(TypeSubTypeValue.class, new TypeSubTypeValueKryoSerializer());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Test
public void testRegistered() {
    // Entity
    assertEquals(EntityKryoSerializer.class, kryo.getSerializer(Entity.class).getClass());

    // Edge
    assertEquals(EdgeKryoSerializer.class, kryo.getSerializer(Edge.class).getClass());

    // Properties
    assertTrue(kryo.getRegistration(Properties.class).getId() > 0);

    // FreqMap
    assertEquals(FreqMapKryoSerializer.class, kryo.getSerializer(FreqMap.class).getClass());

    // HyperLogLogPlus
    assertEquals(HyperLogLogPlusKryoSerializer.class, kryo.getSerializer(HyperLogLogPlus.class).getClass());

    // TypeValue
    assertEquals(TypeValueKryoSerializer.class, kryo.getSerializer(TypeValue.class).getClass());

    // TypeSubTypeValue
    assertEquals(TypeSubTypeValueKryoSerializer.class, kryo.getSerializer(TypeSubTypeValue.class).getClass());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public boolean test(final HyperLogLogPlus input) {
    if (null == input) {
        return false;
    }
    final long cardinality = input.cardinality();
    if (orEqualTo) {
        if (cardinality <= controlValue) {
            return true;
        }
    } else {
        if (cardinality < controlValue) {
            return true;
        }
    }
    return false;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public HyperLogLogPlus deserialize(final JsonParser jsonParser, final DeserializationContext deserializationContext) throws IOException, JsonProcessingException {

    final TreeNode treeNode = jsonParser.getCodec().readTree(jsonParser);

    final TreeNode coreHyperLogLogPlusObject = treeNode.get("hyperLogLogPlus");
    if (null != coreHyperLogLogPlusObject) {
        final TextNode jsonNodes = (TextNode) coreHyperLogLogPlusObject.get(HyperLogLogPlusJsonConstants.HYPER_LOG_LOG_PLUS_SKETCH_BYTES_FIELD);

        final byte[] nodeAsString = jsonNodes.binaryValue();
        final HyperLogLogPlus hyperLogLogPlus = HyperLogLogPlus.Builder.build(nodeAsString);

        return hyperLogLogPlus;
    } else {
        throw new IllegalArgumentException("Receieved null or empty HyperLogLogPlus sketch");
    }
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Before
public void setup() {
    hyperLogLogPlusWithCardinality5 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 5; i++) {
        hyperLogLogPlusWithCardinality5.offer(i);
    }
    assertEquals(5l, hyperLogLogPlusWithCardinality5.cardinality());

    hyperLogLogPlusWithCardinality15 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 18; i++) {
        hyperLogLogPlusWithCardinality15.offer(i);
    }
    assertEquals(15l, hyperLogLogPlusWithCardinality15.cardinality());

    hyperLogLogPlusWithCardinality31 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 32; i++) {
        hyperLogLogPlusWithCardinality31.offer(i);
    }
    assertEquals(31l, hyperLogLogPlusWithCardinality31.cardinality());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
private void runTestWithSketch(final HyperLogLogPlus sketch) throws IOException {
    // When - serialise
    final String json = mapper.writeValueAsString(sketch);

    // Then - serialise
    final String[] parts = json.split("[:,]");
    final String[] expectedParts = {
            "{\"hyperLogLogPlus\"",
            "{\"hyperLogLogPlusSketchBytes\"",
            "BYTES",
            "\"cardinality\"",
            sketch.cardinality() + "}}"};
    for (int i = 0; i < parts.length; i++) {
        if (2 != i) { // skip checking the bytes
            assertEquals(expectedParts[i], parts[i]);
        }
    }

    // When - deserialise
    final HyperLogLogPlus deserialisedSketch = mapper.readValue(IOUtils.toInputStream(json), HyperLogLogPlus.class);

    // Then - deserialise
    assertNotNull(deserialisedSketch);
    assertEquals(sketch.cardinality(), deserialisedSketch.cardinality());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public long count() {
    final Iterator<HyperLogLogPlus> it = states.iterator();

    if (!it.hasNext()) {
        return 0L;
    }

    HyperLogLogPlus current = it.next();

    while (it.hasNext()) {
        try {
            current.addAll(it.next());
        } catch (CardinalityMergeException e) {
            throw new RuntimeException(e);
        }
    }

    return current.cardinality();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
public CompactionMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
    if (version.hasCompactionAncestors())
    { // skip ancestors
        int nbAncestors = in.readInt();
        in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
    }
    ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
    return new CompactionMetadata(cardinality);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
private Entity createCardinality(final Object source,
                                 final Object destination,
                                 final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);

    return new Entity.Builder()
            .vertex(source)
            .group("cardinality")
            .property("edgeGroup", CollectionUtil.treeSet(edge.getGroup()))
            .property("hllp", hllp)
            .property("count", 1)
            .build();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
private Entity createCardinality(final Object source,
                                 final Object destination,
                                 final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);

    return new Entity.Builder()
            .vertex(source)
            .group("Cardinality")
            .property("edgeGroup", CollectionUtil.treeSet(edge.getGroup()))
            .property("hllp", hllp)
            .property("count", 1L)
            .build();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
@Override
public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
                                            final LeafBucketCollector sub) throws IOException {
    if (valuesSource == null) {
        return LeafBucketCollector.NO_OP_COLLECTOR;
    }
    final BigArrays bigArrays = context.bigArrays();
    final SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
    return new LeafBucketCollectorBase(sub, values) {
        @Override
        public void collect(int doc, long bucket) throws IOException {
            hyperLogLogPlusPlusObjectArray = bigArrays.grow(hyperLogLogPlusPlusObjectArray, bucket + 1);
            values.setDocument(doc);
            final int valuesCount = values.count();
            HyperLogLogPlus hll;
            for (int i = 0; i < valuesCount; i++) {
                hll = deserializeHyperLogLogPlus(values.valueAt(i));
                    HyperLogLogPlus current = hyperLogLogPlusPlusObjectArray.get(bucket);
                    if (current == null) {
                        hyperLogLogPlusPlusObjectArray.set(bucket, hll);
                    } else {
                        try {
                            hyperLogLogPlusPlusObjectArray.set(bucket, (HyperLogLogPlus) hll.merge(current));
                        } catch (CardinalityMergeException cme) {
                            throw new ElasticsearchGenerationException("Failed to merge HyperLogLogPlus structures  ", cme);
                        }
                    }

            }
        }
    };
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
private String getHLLStringForTesting(int max) {
    HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(HyperUniqueSumAggregationBuilder.SERIALIZED_DENSE_PRECISION, HyperUniqueSumAggregationBuilder.SERIALIZED_SPARSE_PRECISION);
    for ( int i = 0; i < max ; i ++){
        hyperLogLogPlus.offer(Integer.toString(i));
    }
    return  Base64.getEncoder().encodeToString(HyperUniqueSumAggregator.serializeHyperLogLogPlus(hyperLogLogPlus).bytes);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入依赖的package包/类
private String getHLLStringForTestingWithWrongPrecision(int max) {
    HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(HyperUniqueSumAggregationBuilder.SERIALIZED_DENSE_PRECISION + 5, HyperUniqueSumAggregationBuilder.SERIALIZED_SPARSE_PRECISION + 5);
    for ( int i = 0; i < max ; i ++){
        hyperLogLogPlus.offer(Integer.toString(i));
    }
    return  Base64.getEncoder().encodeToString(HyperUniqueSumAggregator.serializeHyperLogLogPlus(hyperLogLogPlus).bytes);
}