Java HyperLogLogPlus.offer方法代码示例

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Override
public Iterable<Element> _apply(final String line) {
    final List<Element> elements = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        final HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(8, 8);
        hyperLogLogPlus.offer("B" + i);

        final Entity entity = new Entity.Builder()
                .group("cardinality")
                .vertex("A")
                .property("approxCardinality", hyperLogLogPlus)
                .build();
        elements.add(entity);
    }
    return elements;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Before
public void setup() {
    hyperLogLogPlusWithCardinality5 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 5; i++) {
        hyperLogLogPlusWithCardinality5.offer(i);
    }
    assertEquals(5l, hyperLogLogPlusWithCardinality5.cardinality());

    hyperLogLogPlusWithCardinality15 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 18; i++) {
        hyperLogLogPlusWithCardinality15.offer(i);
    }
    assertEquals(15l, hyperLogLogPlusWithCardinality15.cardinality());

    hyperLogLogPlusWithCardinality31 = new HyperLogLogPlus(5, 5);
    for (int i = 1; i <= 32; i++) {
        hyperLogLogPlusWithCardinality31.offer(i);
    }
    assertEquals(31l, hyperLogLogPlusWithCardinality31.cardinality());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private Entity createCardinality(final Object source,
                                 final Object destination,
                                 final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);

    return new Entity.Builder()
            .vertex(source)
            .group("cardinality")
            .property("edgeGroup", CollectionUtil.treeSet(edge.getGroup()))
            .property("hllp", hllp)
            .property("count", 1)
            .build();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private Entity createCardinality(final Object source,
                                 final Object destination,
                                 final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);

    return new Entity.Builder()
            .vertex(source)
            .group("Cardinality")
            .property("edgeGroup", CollectionUtil.treeSet(edge.getGroup()))
            .property("hllp", hllp)
            .property("count", 1L)
            .build();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private String getHLLStringForTesting(int max) {
    HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(HyperUniqueSumAggregationBuilder.SERIALIZED_DENSE_PRECISION, HyperUniqueSumAggregationBuilder.SERIALIZED_SPARSE_PRECISION);
    for ( int i = 0; i < max ; i ++){
        hyperLogLogPlus.offer(Integer.toString(i));
    }
    return  Base64.getEncoder().encodeToString(HyperUniqueSumAggregator.serializeHyperLogLogPlus(hyperLogLogPlus).bytes);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private String getHLLStringForTestingWithWrongPrecision(int max) {
    HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(HyperUniqueSumAggregationBuilder.SERIALIZED_DENSE_PRECISION + 5, HyperUniqueSumAggregationBuilder.SERIALIZED_SPARSE_PRECISION + 5);
    for ( int i = 0; i < max ; i ++){
        hyperLogLogPlus.offer(Integer.toString(i));
    }
    return  Base64.getEncoder().encodeToString(HyperUniqueSumAggregator.serializeHyperLogLogPlus(hyperLogLogPlus).bytes);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Override
public StringQuality call(StringQuality v1, Writable writable) throws Exception {
    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long empty = v1.getCountEmptyString();
    long alphabetic = v1.getCountAlphabetic();
    long numerical = v1.getCountNumerical();
    long word = v1.getCountWordCharacter();
    long whitespaceOnly = v1.getCountWhitespace();
    HyperLogLogPlus hll = v1.getHll();

    String str = writable.toString();

    if (writable instanceof NullWritable)
        countMissing++;
    else if (meta.isValid(writable))
        valid++;
    else
        invalid++;

    if (str == null || str.isEmpty()) {
        empty++;
    } else {
        if (str.matches("[a-zA-Z]"))
            alphabetic++;
        if (str.matches("\\d+"))
            numerical++;
        if (str.matches("\\w+"))
            word++;
        if (str.matches("\\s+"))
            whitespaceOnly++;
    }

    hll.offer(str);
    return new StringQuality(valid, invalid, countMissing, countTotal, empty, alphabetic, numerical, word,
                    whitespaceOnly, hll);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Override
public HyperLogLogPlus addInput(HyperLogLogPlus acc, InputT record) {
  try {
    acc.offer(CoderUtils.encodeToByteArray(inputCoder, record));
  } catch (CoderException e) {
    throw new IllegalStateException("The input value cannot be encoded: " + e.getMessage(), e);
  }
  return acc;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Test
public void testCoder() throws Exception {
  HyperLogLogPlus hllp = new HyperLogLogPlus(12, 18);
  for (int i = 0; i < 10; i++) {
    hllp.offer(i);
  }
  CoderProperties.<HyperLogLogPlus>coderDecodeEncodeEqual(
      ApproximateDistinct.HyperLogLogPlusCoder.of(), hllp);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
protected Entity createCardinality(final Object source,
                                   final Object destination,
                                   final Edge edge) {
    final HyperLogLogPlus hllp = new HyperLogLogPlus(5, 5);
    hllp.offer(destination);

    return new Entity.Builder()
            .vertex(source)
            .group("Cardinality")
            .property("edgeGroup", CollectionUtil.treeSet(edge.getGroup()))
            .property("hllp", hllp)
            .property("count", 1L)
            .build();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Test
public void testConverter() throws ConversionException {
    final HyperLogLogPlus hyperLogLogPlus = new HyperLogLogPlus(5, 5);
    hyperLogLogPlus.offer("A");
    hyperLogLogPlus.offer("B");
    assertEquals(hyperLogLogPlus.cardinality(), (long) HYPER_LOG_LOG_PLUS_CONVERTER.convert(hyperLogLogPlus));

    final HyperLogLogPlus emptyHyperLogLogPlus = new HyperLogLogPlus(5, 5);
    assertEquals(emptyHyperLogLogPlus.cardinality(), (long) HYPER_LOG_LOG_PLUS_CONVERTER.convert(emptyHyperLogLogPlus));
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private static List<Element> getElementsWithNonStandardProperties() {
    final List<Element> elements = new ArrayList<>();

    final FreqMap freqMap = new FreqMap();
    freqMap.put("W", 10L);
    freqMap.put("X", 100L);

    final HyperLogLogPlus hllpp = new HyperLogLogPlus(5, 5);
    hllpp.offer("AAA");

    final Entity entity = new Entity.Builder()
            .group(TestGroups.ENTITY)
            .vertex("A")
            .property("freqMap", freqMap)
            .property("hllpp", hllpp)
            .build();
    elements.add(entity);

    final Edge edge = new Edge.Builder()
            .group(TestGroups.EDGE)
            .source("B")
            .dest("C")
            .directed(true)
            .build();
    final FreqMap freqMap2 = new FreqMap();
    freqMap2.put("Y", 1000L);
    freqMap2.put("Z", 10000L);
    edge.putProperty("freqMap", freqMap2);
    final HyperLogLogPlus hllpp2 = new HyperLogLogPlus(5, 5);
    hllpp2.offer("AAA");
    hllpp2.offer("BBB");
    edge.putProperty("hllpp", hllpp2);
    elements.add(edge);
    return elements;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private static List<Element> getElementsForUserDefinedConversion() {
    final List<Element> elements = new ArrayList<>();

    final FreqMap freqMap = new FreqMap();
    freqMap.put("W", 10L);
    freqMap.put("X", 100L);

    final HyperLogLogPlus hllpp = new HyperLogLogPlus(5, 5);
    hllpp.offer("AAA");

    final Entity entity = new Entity.Builder()
            .group(TestGroups.ENTITY)
            .vertex("A")
            .property("freqMap", freqMap)
            .property("hllpp", hllpp)
            .property("myProperty", new MyProperty(10))
            .build();
    elements.add(entity);

    final Edge edge = new Edge.Builder()
            .group(TestGroups.EDGE)
            .source("B")
            .dest("C")
            .directed(true)
            .build();

    final FreqMap freqMap2 = new FreqMap();
    freqMap2.put("Y", 1000L);
    freqMap2.put("Z", 10000L);
    edge.putProperty("freqMap", freqMap2);

    final HyperLogLogPlus hllpp2 = new HyperLogLogPlus(5, 5);
    hllpp2.offer("AAA");
    hllpp2.offer("BBB");
    edge.putProperty("hllpp", hllpp2);
    edge.putProperty("myProperty", new MyProperty(50));
    elements.add(edge);
    return elements;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
private void setupHllp(final int p, final int sp) {
    hyperLogLogPlus1 = new HyperLogLogPlus(p, sp);
    hyperLogLogPlus1.offer("A");
    hyperLogLogPlus1.offer("B");

    hyperLogLogPlus2 = new HyperLogLogPlus(p, sp);
    hyperLogLogPlus2.offer("C");
    hyperLogLogPlus2.offer("D");
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; //导入方法依赖的package包/类
@Test
public void testValidHyperLogLogPlusSketchSerialisedCorrectly() throws IOException {
    // Given
    final String testString = "TestString";

    final HyperLogLogPlus sketch = new HyperLogLogPlus(5, 5);
    sketch.offer(testString);

    // When / Then
    runTestWithSketch(sketch);
}