Java Clip.getRecords方法代码示例

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testSizeLimiting() {
    FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 10);
    // For i from 1 to 13, update the sketch i times
    IntStream.range(1, 13).forEach(i -> IntStream.range(0, i).forEach(j -> sketch.update(String.valueOf(i))));

    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 10);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 2);
        Integer item = Integer.valueOf(actual.get(FrequentItemsSketch.ITEM_FIELD).toString());
        // 1, 2 had the lowest and since our size is 10, we should have not seen them
        Assert.assertTrue(item > 2 && item < 13);
        Assert.assertEquals(actual.get(FrequentItemsSketch.COUNT_FIELD), Long.valueOf(item));
    }
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testNoDataQuantileDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, DistributionType.QUANTILE, new double[]{ 0, 0.3, 1 });

    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);

    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);

    BulletRecord expectedA = RecordBox.get().add(QUANTILE_FIELD, 0.0).add(VALUE_FIELD, Double.POSITIVE_INFINITY)
                                            .getRecord();
    BulletRecord expectedB = RecordBox.get().add(QUANTILE_FIELD, 0.3).add(VALUE_FIELD, Double.NaN).getRecord();
    BulletRecord expectedC = RecordBox.get().add(QUANTILE_FIELD, 1.0).add(VALUE_FIELD, Double.NEGATIVE_INFINITY)
                                            .getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testRounding() {
    Distribution distribution = makeDistribution(DistributionType.QUANTILE, 20, 6, 0.0, 1.0, 0.1);

    IntStream.range(0, 10).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord())
                           .forEach(distribution::consume);

    Clip result = distribution.getAggregation();

    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 11);
    Set<String> actualQuantilePoints = records.stream().map(r -> r.get(QUANTILE_FIELD).toString())
                                                       .collect(Collectors.toSet());
    Set<String> expectedQuantilePoints = new HashSet<>(Arrays.asList("0.0", "0.1", "0.2", "0.3", "0.4", "0.5", "0.6",
                                                                     "0.7", "0.8", "0.9", "1.0"));
    Assert.assertEquals(actualQuantilePoints, expectedQuantilePoints);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testExactTopKSizeLimiting() {
    TopK topK = makeTopK(ErrorType.NO_FALSE_POSITIVES, singletonList("A"), "cnt", 64, 2, null);
    IntStream.range(0, 20).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).forEach(topK::consume);
    IntStream.range(0, 20).mapToObj(i -> RecordBox.get().add("A", 108.1).getRecord()).forEach(topK::consume);
    IntStream.range(0, 100).mapToObj(i -> RecordBox.get().add("A", 42L).getRecord()).forEach(topK::consume);

    Clip result = topK.getAggregation();

    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertFalse((Boolean) metadata.get("isEst"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);

    BulletRecord expectedA = RecordBox.get().add("A", "42").add("cnt", 100L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "108.1").add("cnt", 20L).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testNullAttributes() {
    TopK topK = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 32), null, singletonMap("A", "foo"), 16, null);
    IntStream.range(0, 16).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).forEach(topK::consume);

    Clip result = topK.getAggregation();
    Assert.assertNull(result.getMeta().asMap().get("meta"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 16);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 2);
        int fieldA = Integer.valueOf(actual.get("foo").toString());
        Assert.assertTrue(fieldA < 16);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 1L);
    }
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testRenaming() {
    HashMap<String, String> fields = new HashMap<>();
    fields.put("A", "foo");
    fields.put("fieldB", "");

    TopK topK = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, -1), null, fields, 16, null);
    IntStream.range(0, 16).mapToObj(i -> RecordBox.get().add("A", i).add("fieldB", 32 - i).getRecord())
                          .forEach(topK::consume);

    Clip result = topK.getAggregation();
    Assert.assertNull(result.getMeta().asMap().get("meta"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 16);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 3);
        int fieldA = Integer.valueOf(actual.get("foo").toString());
        int fieldB = Integer.valueOf(actual.get("fieldB").toString());
        Assert.assertTrue(fieldA < 16);
        Assert.assertTrue(fieldB > 16);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 1L);
    }
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testExactTopKThreshold() {
    TopK topK = makeTopK(ErrorType.NO_FALSE_POSITIVES, asList("A", "B"), "cnt", 64, 3, 25L);
    IntStream.range(0, 20).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).forEach(topK::consume);
    IntStream.range(0, 25).mapToObj(i -> RecordBox.get().add("A", 108).getRecord()).forEach(topK::consume);
    IntStream.range(0, 24).mapToObj(i -> RecordBox.get().add("A", "foo").add("B", "bar").getRecord())
                          .forEach(topK::consume);
    IntStream.range(0, 100).mapToObj(i -> RecordBox.get().add("A", 42L).getRecord()).forEach(topK::consume);

    Clip result = topK.getAggregation();

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);

    BulletRecord expectedA = RecordBox.get().add("A", "42").add("B", "null").add("cnt", 100L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "108").add("B", "null").add("cnt", 25L).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);

    IntStream.range(0, 11).mapToObj(i -> RecordBox.get().add("A", "foo").add("B", "bar").getRecord())
                          .forEach(topK::consume);

    result = topK.getAggregation();

    records = result.getRecords();
    Assert.assertEquals(records.size(), 3);

    expectedA = RecordBox.get().add("A", "42").add("B", "null").add("cnt", 100L).getRecord();
    expectedB = RecordBox.get().add("A", "foo").add("B", "bar").add("cnt", 35L).getRecord();
    BulletRecord expectedC = RecordBox.get().add("A", "108").add("B", "null").add("cnt", 25L).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testCasting() {
    Distribution distribution = makeDistribution(DistributionType.PMF, Collections.singletonList(50.0));

    IntStream.range(0, 25).mapToObj(String::valueOf).map(s -> RecordBox.get().add("field", s).getRecord())
                          .forEach(distribution::consume);

    distribution.consume(RecordBox.get().add("field", "garbage").getRecord());
    distribution.consume(RecordBox.get().add("field", "1.0 garbage").getRecord());


    IntStream.range(50, 100).mapToDouble(i -> i).mapToObj(d -> RecordBox.get().add("field", d).getRecord())
                            .forEach(distribution::consume);

    Clip result = distribution.getAggregation();

    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);

    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 50.0 + END_EXCLUSIVE)
                                            .add(COUNT_FIELD, 25.0)
                                            .add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 50.0 + SEPARATOR + POSITIVE_INFINITY_END)
                                            .add(COUNT_FIELD, 50.0)
                                            .add(PROBABILITY_FIELD, 2.0 / 3).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testExactTopK() {
    TopK topK = makeTopK(asList("A", "B"), 64, 20);
    IntStream.range(0, 996).mapToObj(i -> RecordBox.get().add("A", String.valueOf(i % 3)).add("B", i % 4).getRecord())
                           .forEach(topK::consume);

    Clip result = topK.getAggregation();

    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals((String) metadata.get("family"), Family.FREQUENCY.getFamilyName());
    Assert.assertEquals(metadata.get("error"), 0L);
    Assert.assertEquals(metadata.get("n"), 996L);
    Assert.assertEquals(metadata.get("actives"), 12);

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 12);
    Set<Integer> fields = new HashSet<>();
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 3);
        int fieldA = Integer.valueOf(actual.get("A").toString());
        int fieldB = Integer.valueOf(actual.get("B").toString());
        Assert.assertTrue(fieldA < 3);
        Assert.assertTrue(fieldB < 4);
        fields.add(fieldA * 3 + fieldB * 4);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 83L);
    }
    Assert.assertEquals(fields.size(), 12);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testResetting() {
    FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 32);
    IntStream.range(0, 10).forEach(i -> IntStream.range(0, 10).forEach(j -> sketch.update(String.valueOf(i))));
    IntStream.range(10, 100).forEach(i -> sketch.update("bar"));

    FrequentItemsSketch another = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 32);
    another.update("foo");
    another.update("bar");
    another.update("baz");

    sketch.union(another.serialize());

    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(result.getRecords().size(), 13);

    sketch.reset();

    result = sketch.getResult("meta", ALL_METADATA);
    metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals((String) metadata.get("family"), Family.FREQUENCY.getFamilyName());
    Assert.assertNull(metadata.get("size"));
    Assert.assertEquals(metadata.get("error"), 0L);
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("actives"), 0);
    Assert.assertFalse((Boolean) metadata.get("isEst"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 0);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testNoDataPMFDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, 2, DistributionType.PMF, 10);

    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);

    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);

    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + Double.POSITIVE_INFINITY + END_EXCLUSIVE)
                                            .add(PROBABILITY_FIELD, Double.NaN)
                                            .add(COUNT_FIELD, Double.NaN)
                                            .getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + Double.POSITIVE_INFINITY + SEPARATOR + POSITIVE_INFINITY_END)
                                            .add(PROBABILITY_FIELD, Double.NaN)
                                            .add(COUNT_FIELD, Double.NaN)
                                            .getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testNoDataCDFDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, 2, DistributionType.CDF, 10);

    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);

    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);

    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + Double.POSITIVE_INFINITY + END_EXCLUSIVE)
                                            .add(PROBABILITY_FIELD, Double.NaN)
                                            .add(COUNT_FIELD, Double.NaN)
                                            .getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END)
                                            .add(PROBABILITY_FIELD, Double.NaN)
                                            .add(COUNT_FIELD, Double.NaN)
                                            .getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testBadMaxMapEntries() {
    TopK topK = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, -1), null, singletonMap("A", "foo"),
                         BulletConfig.DEFAULT_TOP_K_AGGREGATION_SKETCH_ENTRIES, null);
    int uniqueGroups = BulletConfig.DEFAULT_TOP_K_AGGREGATION_SKETCH_ENTRIES / 4;

    IntStream.range(0, uniqueGroups).mapToObj(i -> RecordBox.get().add("A", i).getRecord())
                                    .forEach(topK::consume);

    Clip result = topK.getAggregation();
    Assert.assertNull(result.getMeta().asMap().get("meta"));

    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), uniqueGroups);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 2);
        int fieldA = Integer.valueOf(actual.get("foo").toString());
        Assert.assertTrue(fieldA < uniqueGroups);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 1L);
    }

    // Not a power of 2
    TopK another = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 5), null, singletonMap("A", "foo"),
                            BulletConfig.DEFAULT_TOP_K_AGGREGATION_SKETCH_ENTRIES, null);
    IntStream.range(0, uniqueGroups).mapToObj(i -> RecordBox.get().add("A", i).getRecord())
                                    .forEach(another::consume);

    result = another.getAggregation();
    Assert.assertNull(result.getMeta().asMap().get("meta"));

    records = result.getRecords();
    Assert.assertEquals(records.size(), uniqueGroups);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testDistincts() {
    List<String> fields = asList("fieldA", "fieldB", "fieldC");
    GroupBy groupBy = makeDistinct(fields, 3);

    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("fieldC", "baz").getRecord();
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordA));

    BulletRecord recordB = RecordBox.get().add("fieldA", "1").add("fieldB", "2").getRecord();
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordB));

    groupBy.consume(RecordBox.get().getRecord());

    Clip aggregate = groupBy.getAggregation();
    Assert.assertNotNull(aggregate);

    Map<String, Object> meta = aggregate.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);

    List<BulletRecord> records = aggregate.getRecords();

    Assert.assertEquals(records.size(), 3);

    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("fieldC", "baz").getRecord();
    BulletRecord expectedB = RecordBox.get().add("fieldA", "1").add("fieldB", "2").add("fieldC", "null").getRecord();
    BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add("fieldC", "null").getRecord();

    // We have each distinct record exactly once
    assertContains(records, expectedA);
    assertContains(records, expectedB);
    assertContains(records, expectedC);
}
 

import com.yahoo.bullet.result.Clip; //导入方法依赖的package包/类
@Test
public void testGroupByOperations() {
    List<String> fields = asList("fieldA", "fieldB");
    GroupBy groupBy = makeGroupBy(fields, 3, makeGroupOperation(COUNT, null, null),
                                  makeGroupOperation(SUM, "price", "priceSum"));

    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordB = RecordBox.get().addNull("fieldA").add("fieldB", "bar").add("price", 1).getRecord();

    IntStream.range(0, 10).forEach(i -> groupBy.consume(recordA));
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordB));
    IntStream.range(0, 20).forEach(i -> groupBy.consume(recordA));
    groupBy.consume(recordB);

    Clip aggregate = groupBy.getAggregation();
    Assert.assertNotNull(aggregate);

    Map<String, Object> meta = aggregate.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);

    List<BulletRecord> records = aggregate.getRecords();

    Assert.assertEquals(records.size(), 2);

    // count = 10 + 20, price = 10*3 + 20*3
    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar")
                                            .add(COUNT.getName(), 30L).add("priceSum", 90.0).getRecord();
    // count = 9 + 1, price = 9*1 + 1*1
    BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar")
                                            .add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();

    assertContains(records, expectedA);
    assertContains(records, expectedB);
}