Java HyperLogLog类代码示例

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public void add() {
  if (work.obj != null) {
    com.clearspring.analytics.stream.cardinality.HyperLogLog hll =
      (com.clearspring.analytics.stream.cardinality.HyperLogLog) work.obj;


    if (in.isSet == 0) { // No hll structure to merge
      return;
    }
    // fall through //
    try {
      byte[] buf = com.dremio.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(
        in.start, in.end, in.buffer).getBytes();
      com.clearspring.analytics.stream.cardinality.HyperLogLog other =
        com.clearspring.analytics.stream.cardinality.HyperLogLog.Builder.build(buf);
      hll.addAll(other);
    } catch (Exception e) {
      throw new java.lang.RuntimeException("Failed to merge HyperLogLog output", e);
    }
    work.obj = null;
  }
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public void output() {
  if (work.obj != null) {
    com.clearspring.analytics.stream.cardinality.HyperLogLog hll =
      (com.clearspring.analytics.stream.cardinality.HyperLogLog) work.obj;

    try {
      byte[] ba = hll.getBytes();
      buffer = buffer.reallocIfNeeded(ba.length);
      out.buffer = buffer;
      out.start = 0;
      out.end = ba.length;
      out.buffer.setBytes(0, ba);
      out.isSet = 1;
    } catch (java.io.IOException e) {
      throw new java.lang.RuntimeException("Failed to get HyperLogLog output", e);
    }
  } else {
    out.isSet = 0;
  }
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
/** Loads a segment from a persistent snapshot. */
Segment(UUID id, Snapshot snapshot, long splitThresholdBytes, SplitQueue<Segment> splitQueue) {
    // Fail if the segment was written with a newer incompatible data format.
    if (snapshot.version > PERSISTENCE_VERSION) {
        throw new UnsupportedOperationException("Unsupported persistent sorted queue data version: " + snapshot.version);
    }
    _id = checkNotNull(id, "id");
    _dataId = Objects.firstNonNull(snapshot.dataId, id);  // dataId should be non-null except for segments before dataId was introduced
    _min = (snapshot.min != null) ? ByteBufferUtil.hexToBytes(snapshot.min) : null;
    _adds = snapshot.adds;
    _bytesAdded = snapshot.bytesAdded;
    try {
        _distinctAdds = HyperLogLog.Builder.build(checkNotNull(snapshot.distinctAddsHll, "distinctAddsHll"));
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
    _deletes = snapshot.deletes;
    _bytesUntilSplitCheckSize = snapshot.bytesUntilSplitCheckSize;
    _bytesUntilSplitCheckRemaining = 0;
    _splitThresholdBytes = splitThresholdBytes;
    _splitting = snapshot.splitting;
    _splitTargetSize = snapshot.splitTargetSize;
    _splitTargetRemaining = snapshot.splitTargetRemaining;
    _splitQueue = checkNotNull(splitQueue, "splitQueue");
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
/** Verify we can load data from when segments had id == dataId so dataId was missing from the snapshot. */
@Test
@SuppressWarnings("unchecked")
public void testOldSnapshotMigration() throws IOException {
    SplitQueue<Segment> splitQueue = mock(SplitQueue.class);

    String hllState = Base64.encodeBase64String(new HyperLogLog(Segment.HLL_LOG2M).getBytes());
    String json = "{\"version\":1,\"min\":null,\"adds\":1,\"bytesAdded\":20000," +
            "\"distinctAddsHll\":\"" + hllState + "\",\"deletes\":0,\"bytesUntilSplitCheckSize\":771," +
            "\"splitting\":true,\"splitTargetSize\":5500,\"splitTargetRemaining\":4266}";

    // Test decoding.
    Segment.Snapshot actual = JsonHelper.fromJson(json, Segment.Snapshot.class);

    // Verify we can create a segment that looks like it's in the same state as the original.
    UUID id = TimeUUIDs.newUUID();
    Segment hydrated = new Segment(id, actual, 12345, splitQueue);
    assertEquals(hydrated.getId(), id);
    assertEquals(hydrated.getDataId(), id);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public long count() {
    final Iterator<HyperLogLog> it = states.iterator();

    if (!it.hasNext()) {
        return 0L;
    }

    HyperLogLog current = it.next();

    while (it.hasNext()) {
        try {
            current.addAll(it.next());
        } catch (CardinalityMergeException e) {
            throw new RuntimeException(e);
        }
    }

    return current.cardinality();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public List<HyperLogLog> combine(List<HyperLogLog> aggregationResultList, CombineLevel combineLevel) {
  if ((aggregationResultList == null) || aggregationResultList.isEmpty()) {
    return null;
  }
  HyperLogLog hllResult = aggregationResultList.get(0);
  for (int i = 1; i < aggregationResultList.size(); ++i) {
    try {
      hllResult.addAll(aggregationResultList.get(i));
    } catch (CardinalityMergeException e) {
      LOGGER.error("Caught exception while merging Cardinality using HyperLogLog", e);
      Utils.rethrowException(e);
    }
  }
  aggregationResultList.clear();
  aggregationResultList.add(hllResult);
  return aggregationResultList;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public HyperLogLog combineTwoValues(HyperLogLog aggregationResult0, HyperLogLog aggregationResult1) {
  if (aggregationResult0 == null) {
    return aggregationResult1;
  }
  if (aggregationResult1 == null) {
    return aggregationResult0;
  }
  try {
    aggregationResult0.addAll(aggregationResult1);
  } catch (CardinalityMergeException e) {
    LOGGER.error("Caught exception while merging Cardinality using HyperLogLog", e);
    Utils.rethrowException(e);
  }
  return aggregationResult0;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public Long reduce(List<HyperLogLog> combinedResultList) {
  if ((combinedResultList == null) || combinedResultList.isEmpty()) {
    return 0L;
  }
  HyperLogLog reducedResult = combinedResultList.get(0);
  for (int i = 1; i < combinedResultList.size(); ++i) {
    try {
      reducedResult.addAll(combinedResultList.get(i));
    } catch (CardinalityMergeException e) {
      LOGGER.error("Caught exception while merging Cardinality using HyperLogLog", e);
      Utils.rethrowException(e);
    }
  }
  return reducedResult.cardinality();
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Test
public void testInsertionTime() {
    int numOfItems = 10000000;

    println("#items_inserted, HLL_time(nano), IntOpenHashSet(nano), time_ratio, estimate, precise, error");
    for (int i = 0; i < numOfItems; i+=numOfItems/17) {
        if (i == 0) {
            continue;
        }
        RandomNumberArray arr = new RandomNumberArray(i, DUPLICATION_PER_ITEM);
        HyperLogLog hllResult = new HyperLogLog(DistinctCountHLLAggregationFunction.DEFAULT_BIT_SIZE);
        IntOpenHashSet set = new IntOpenHashSet(); //HashSet<Integer> set = new HashSet<Integer>();
        long t1 = System.nanoTime();
        arr.offerAllNumberTo(hllResult);
        long t2 = System.nanoTime();
        arr.offerAllNumberTo(set);
        long t3 = System.nanoTime();
        long estimate = hllResult.cardinality();
        long precise = set.size();
        println(i + ", " + "" + (t2 - t1) + ", " + (t3 - t2) + ", " + (t2 - t1 + 0.0) / (t3 - t2 + 0.0) + ", "
                + estimate + ", " + precise + ", " + getErrorString(precise, estimate));
    }
    assertEquals(true, true);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Test
public void testMemoryConsumption() {
    int numOfItems = 10000000;

    println("#items_inserted, HLL_ser_size, openHashSet_ser_size, ser_size_ratio, estimate, precise, error");
    for (int i = 0; i < numOfItems; i+=numOfItems/17) {
        if (i == 0) {
            continue;
        }
        RandomNumberArray arr = new RandomNumberArray(i, DUPLICATION_PER_ITEM);
        HyperLogLog hllResult = new HyperLogLog(DistinctCountHLLAggregationFunction.DEFAULT_BIT_SIZE);
        IntOpenHashSet set = new IntOpenHashSet();
        arr.offerAllNumberTo(hllResult);
        arr.offerAllNumberTo(set);
        int hllSize = getSerializedSize(hllResult);
        int setSize = getSerializedSize(set);
        long estimate = hllResult.cardinality();
        long precise = set.size();
        println(i + ", " + hllSize + ", " + setSize + ", " + (hllSize + 0.0) / (setSize + 0.0) + ", "
                + estimate + ", " + precise + ", " + getErrorString(precise, estimate));
    }
    assertEquals(true, true);
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Test
public void hllSerializationTest() throws IOException {

    SiteSession siteSession = new SiteSession("user1", 100, "testURL");
    siteSession.update(200, "testURL2");
    siteSession.update(300, "testURL");
    siteSession.update(400, "testURL2");

    long inputCardinality = siteSession.getHyperLogLog().cardinality();

    byte[] inputBytes = siteSession.getHyperLogLog().getBytes();
    ByteBuffer byteBuffer = ByteBuffer.allocate(4+inputBytes.length);
    byteBuffer.putInt(inputBytes.length);
    byteBuffer.put(inputBytes);
    byteBuffer.flip();

    byte[] rebuiltBytes = new byte[byteBuffer.getInt()];
    byteBuffer.get(rebuiltBytes);
    HyperLogLog rebuiltHll = HyperLogLog.Builder.build(rebuiltBytes);

    assertEquals(inputCardinality, rebuiltHll.cardinality());
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
public HyperLogLog fetchSketchForMin(final int minOfDay){
    System.out.println("Launch DRPC query for " +  minOfDay);
    //launch DRPC request
    //here it is immediate
    //TODO: support Async here
    String rtnVals = StormClusterStore.getInstance().getLocalDRPC().
            execute(UniqueUserCounterTopologyBuilder.DRPC_STREAM_NAME,
                    String.valueOf(minOfDay));


    try {
        HyperLogLog hll = HyperLogLog.Builder.build(Base64.decodeBase64(rtnVals));
        System.out.println("unique items for string " + hll.cardinality());
        return hll;
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    byte[] sketchBytes = null;
    switch (sketchType) {

        case HLL:
            HyperLogLog hyperLogLog = (HyperLogLog) tuple.getValueByField(fieldName);

            try {
                sketchBytes = hyperLogLog.getBytes();
            } catch (IOException e) {
                e.printStackTrace();
            }
            break;
        case CMS:

            CountMinSketch cms = (CountMinSketch) tuple.getValueByField(fieldName);
            sketchBytes = CountMinSketch.serialize(cms);
            break;
    }

    String sketchAsStr = Base64.encodeBase64String(sketchBytes);
    collector.emit(new Values(sketchAsStr));
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
@Test
public void hllSerializationTest() {
    HyperLogLog hll = new HyperLogLog(16);
    for (int i = 0; i < 10; i++) {
        hll.offer(i);
    }

    try {

        //this would check if we can return the data as string
        //and still be usable after we build HLL
        //both should use the same bytes and same hash function to merge cardinality

        String hllAsStr = Base64.encodeBase64String(hll.getBytes());

        HyperLogLog hll2 = HyperLogLog.Builder.build(Base64.decodeBase64(hllAsStr));
        assertEquals(hll.cardinality(), hll2.cardinality());
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

import com.clearspring.analytics.stream.cardinality.HyperLogLog; //导入依赖的package包/类
public MetricBuffer(MetricBuffer copy) {
  this.values = new Object[copy.values.length];
  for (int i = 0; i < this.values.length; i++) {
    Object copyValue = copy.values[i];
    if (copyValue instanceof HyperLogLog) {
      // deep copy of hll field
      this.values[i] = HllUtil.clone((HyperLogLog)copyValue,
          HllSizeUtils.getLog2mFromHllFieldSize(copy.metricFieldSpecs.get(i).getFieldSize()));
    } else if (copyValue instanceof Number) {
      // number field is immutable
      this.values[i] = copyValue;
    } else {
      throw new IllegalArgumentException("Unsupported metric type: " + copyValue.getClass());
    }
  }
  this.metricFieldSpecs = copy.metricFieldSpecs;
}