Java Aggregate类代码示例

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
/**
 * Creates an {@link Aggregate} from the given {@link InputEvent}.
 *
 * @param inputEvent      The {@link InputEvent} to unpack into an {@link Aggregate}.
 * @param context         The conversion context required to transform the {@link InputEvent} into
 *                        the correct {@link Aggregate}.
 * @param aggregatorIndex The aggregatorIndex assigned to this {@link Aggregate}.
 * @return The converted {@link Aggregate}.
 */
public static Aggregate createAggregate(InputEvent inputEvent,
    DimensionsConversionContext context,
    int aggregatorIndex)
{
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  EventKey eventKey = createEventKey(inputEvent,
      context,
      aggregatorIndex);

  Aggregate aggregate = new Aggregate(eventKey,
      aggregates);
  aggregate.setAggregatorIndex(aggregatorIndex);

  return aggregate;
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  GPOMutable keys = new GPOMutable(context.keyDescriptor);
  GPOUtils.indirectCopy(keys, src.getKeys(), context.indexSubsetKeys);

  EventKey eventKey = createEventKey(src,
      context,
      aggregatorIndex);

  long[] longFields = aggregates.getFieldsLong();

  for (int index = 0;
      index < longFields.length;
      index++) {
    longFields[index] = 0;
  }

  return new Aggregate(eventKey,
      aggregates);
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
/**
 * The result keep a list of object for each aggregate value
 * The value of resultAggregate should keep a list of inputEventKey(the value can be get from cache or load) or a map
 * from inputEventKey to the value instead of just a list of aggregate value. As the value could be changed in
 * current window, and this change should be applied.
 *
 * precondition: resultAggregate.eventKey matches with inputSubEventKeys
 * notes: this algorithm only support TOP for positive values and BOTTOM for negative values
 */
@Override
public void aggregate(Aggregate resultAggregate, Set<EventKey> inputSubEventKeys,
    Map<EventKey, Aggregate> inputAggregatesRepo)
{
  //there are problem for composite's value field descriptor, just ignore now.
  GPOMutable resultGpo = resultAggregate.getAggregates();
  final List<String> compositeFieldList = resultAggregate.getEventKey().getKey().getFieldDescriptor().getFieldList();

  //Map<EventKey, Aggregate> existedSubEventKeyToAggregate = Maps.newHashMap();
  for (String valueField : resultGpo.getFieldDescriptor().getFieldList()) {
    //the resultGpo keep a list of sub aggregates
    updateAggregate(resultAggregate, valueField, inputSubEventKeys, inputAggregatesRepo);

    //compare the existed sub aggregates with the new input aggregates to update the list
    for (EventKey eventKey : inputSubEventKeys) {
      aggregate(compositeFieldList, resultGpo, eventKey, inputAggregatesRepo.get(eventKey).getAggregates());
    }
  }

}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
/**
 * update existed sub aggregate.
 * The sub aggregates which kept in composite aggregate as candidate could be changed. synchronize the value with
 * input aggregates.
 *
 * @param resultAggregate
 * @param valueField
 * @param inputSubEventKeys
 * @param inputAggregatesRepo
 */
@SuppressWarnings("unchecked")
protected void updateAggregate(Aggregate resultAggregate, String valueField,
    Set<EventKey> inputSubEventKeys, Map<EventKey, Aggregate> inputAggregatesRepo)
{
  Map<String, Object> resultAggregateFieldToValue =
      (Map<String, Object>)resultAggregate.getAggregates().getFieldObject(valueField);
  if (resultAggregateFieldToValue == null) {
    return;
  }

  for (EventKey inputSubEventKey : inputSubEventKeys) {
    Aggregate inputSubAggregate = inputAggregatesRepo.get(inputSubEventKey);
    String mapKey = getStoreMapKey(inputSubAggregate.getEventKey(),
        resultAggregate.getEventKey().getKey().getFieldDescriptor().getFieldList());
    //Aggregate existedAggregate = existedSubEventKeyToAggregate.get(inputSubEventKey);
    if (resultAggregateFieldToValue.get(mapKey) != null) {
      resultAggregateFieldToValue.put(mapKey, inputSubAggregate.getAggregates().getField(valueField));
    }
  }
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Override
public Collection<Partition<AbstractSinglePortHDHTWriter<Aggregate>>>
    definePartitions(Collection<Partition<AbstractSinglePortHDHTWriter<Aggregate>>> partitions,
    PartitioningContext context)
{
  Collection<Partition<AbstractSinglePortHDHTWriter<Aggregate>>> newPartitions = super.definePartitions(partitions,
      context);

  if (newPartitions.size() == partitions.size()) {
    return newPartitions;
  }

  Iterator<Partition<AbstractSinglePortHDHTWriter<Aggregate>>> iterator = newPartitions.iterator();

  long bucket = ((AppDataSingleSchemaDimensionStoreHDHT)iterator.next().getPartitionedInstance()).getBucketID();
  long last = bucket + newPartitions.size();

  for (; ++bucket < last;) {
    ((AppDataSingleSchemaDimensionStoreHDHT)iterator.next().getPartitionedInstance()).setBucketID(bucket);
  }

  return newPartitions;
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
/**
 * This method serializes the aggregate payload ({@link GPOMutable}) in the given {@link Aggregate}.
 *
 * @param event The {@link Aggregate} whose aggregate payload needs to be serialized.
 * @return The serialized aggregate payload of the given {@link Aggregate}.
 */
public synchronized byte[] getValueBytesGAE(Aggregate event)
{
  FieldsDescriptor metaDataDescriptor = null;
  final int aggregatorID = event.getEventKey().getAggregatorID();
  if (getAggregator(aggregatorID) != null) {
    metaDataDescriptor = getAggregator(aggregatorID).getMetaDataDescriptor();
  } else {
    metaDataDescriptor = getCompositeAggregator(aggregatorID).getMetaDataDescriptor();
  }

  if (metaDataDescriptor != null) {
    bal.add(GPOUtils.serialize(event.getMetaData(), tempBal));
  }

  bal.add(GPOUtils.serialize(event.getAggregates(), tempBal));
  byte[] serializedBytes = bal.toByteArray();
  bal.clear();

  return serializedBytes;
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
/**
 * The impressions and cost could be SUM or COUNT
 * @param aggregatorType
 * @param location
 * @param impressions
 * @param cost
 * @return
 */
public Aggregate createEvent(
    AggregatorIncrementalType aggregatorType,
    String location,
    Long impressions,
    Object cost)
{
  return createEvent(eventSchema,
                   aggregatorType,
                   publisher,
                   location,
                   60000L,
                   TimeBucket.MINUTE,
                   impressions,
                   cost);
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Test
public void complexOutputTest()
{
  AdInfo ai = createTestAdInfoEvent1();

  DimensionsComputationFlexibleSingleSchemaPOJO dcss =
      createDimensionsComputationOperator("adsGenericEventSimpleTopBottom.json");

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<Aggregate>();
  TestUtils.setSink(dcss.output, sink);

  dcss.setup(null);
  dcss.beginWindow(0L);
  dcss.input.put(ai);
  dcss.endWindow();

  Assert.assertEquals(4, sink.collectedTuples.size());
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Test
public void complexOutputTest()
{
  AdInfo ai = createTestAdInfoEvent1();

  DimensionsComputationFlexibleSingleSchemaPOJO dcss =
      createDimensionsComputationOperator("adsGenericEventSchemaAdditional.json");

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<Aggregate>();
  TestUtils.setSink(dcss.output, sink);

  dcss.setup(null);
  dcss.beginWindow(0L);
  dcss.input.put(ai);
  dcss.endWindow();

  Assert.assertEquals(60, sink.collectedTuples.size());
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Test
public void complexCornerCase()
{
  AdInfo ai = createTestAdInfoEvent1();
  AdInfo ai2 = createTestAdInfoEvent2();
  AdInfo ai3 = createTestAdInfoEvent3();

  DimensionsComputationFlexibleSingleSchemaPOJO dcss =
      createDimensionsComputationOperator("adsGenericEventSchemaAdditionalCorner.json");

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<Aggregate>();
  TestUtils.setSink(dcss.output, sink);

  dcss.setup(null);
  dcss.beginWindow(0L);
  dcss.input.put(ai);
  dcss.input.put(ai2);
  dcss.input.put(ai3);
  dcss.endWindow();

  Assert.assertEquals(6, sink.collectedTuples.size());
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Test
public void aggregationsTest()
{
  AdInfo ai = createTestAdInfoEvent1();
  AdInfo ai2 = createTestAdInfoEvent2();

  DimensionsComputationFlexibleSingleSchemaPOJO dcss =
      createDimensionsComputationOperator("adsGenericEventSchemaAggregations.json");

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<Aggregate>();
  TestUtils.setSink(dcss.output, sink);

  dcss.setup(null);
  dcss.beginWindow(0L);
  dcss.input.put(ai);
  dcss.input.put(ai2);
  dcss.endWindow();

  Assert.assertEquals(6, sink.collectedTuples.size());
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Test
public void differentTimeBucketsForDimensionCombinations()
{
  AdInfo ai = createTestAdInfoEvent1();

  DimensionsComputationFlexibleSingleSchemaPOJO dcss =
      createDimensionsComputationOperator("adsGenericEventSchemaDimensionTimeBuckets.json");

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<>();
  TestUtils.setSink(dcss.output, sink);

  dcss.setup(null);
  dcss.beginWindow(0L);
  dcss.input.put(ai);
  dcss.endWindow();

  Assert.assertEquals(11, sink.collectedTuples.size());
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);
  return aggregate;
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate agg = createAggregate(src,
      context,
      aggregatorIndex);

  GPOUtils.indirectCopy(agg.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  GPOMutable metaData = new GPOMutable(getMetaDataDescriptor());

  GPOMutable fullKey = new GPOMutable(src.getKeys());

  if (context.inputTimestampIndex >= 0) {
    fullKey.getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  List<GPOMutable> keys = Lists.newArrayList(fullKey);

  GPOMutable value = new GPOMutable(agg.getAggregates());
  List<GPOMutable> values = Lists.newArrayList(value);

  metaData.getFieldsObject()[KEY_FD_INDEX] = fullKey.getFieldDescriptor();
  metaData.getFieldsObject()[AGGREGATE_FD_INDEX] = value.getFieldDescriptor();
  metaData.getFieldsObject()[KEYS_INDEX] = keys;
  metaData.getFieldsObject()[AGGREGATES_INDEX] = values;
  agg.setMetaData(metaData);

  return agg;
}
 

import org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate; //导入依赖的package包/类
@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  @SuppressWarnings("unchecked")
  List<GPOMutable> destKeys =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[KEYS_INDEX];

  @SuppressWarnings("unchecked")
  List<GPOMutable> destAggregates =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[AGGREGATES_INDEX];

  long timestamp = 0L;

  if (context.inputTimestampIndex >= 0) {
    timestamp = src.getKeys().getFieldsLong()[context.inputTimestampIndex];
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  if (!contains(destKeys, src.getKeys())) {
    destKeys.add(new GPOMutable(src.getKeys()));

    GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
    GPOUtils.indirectCopy(aggregates, src.getAggregates(), context.indexSubsetAggregates);

    destAggregates.add(aggregates);

    this.aggregateAggs(dest.getAggregates(), aggregates);
  }

  if (context.inputTimestampIndex >= 0) {
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = timestamp;
  }
}