Java AfterProcessingTime类代码示例

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

    Options options =
        PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
    Pipeline pipeline = Pipeline.create(options);

    pipeline
    .apply(KafkaIO.<String, String>read()
        .withBootstrapServers(options.getKafkaBootstrapServer())
        .withTopic(options.getTopic())
        .withKeyDeserializer(StringDeserializer.class)
        .withValueDeserializer(StringDeserializer.class)
        .withTimestampFn(new SetTimestampFn()))
    .apply("Values", ParDo.of(new ValuesFn()))

    .apply("FixedWindows", Window.<String>into(FixedWindows.of(FIVE_MINUTES))
        .triggering(AfterWatermark.pastEndOfWindow()
            .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
                .plusDelayOf(TWO_MINUTES))
            .withLateFirings(AfterPane.elementCountAtLeast(1)))
        .withAllowedLateness(TEN_MINUTES)
        .accumulatingFiredPanes())

    .apply("TeamScore", new CalculateTeamScores(options.getOutputPrefix()));

    pipeline.run();
  }
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
  return infos.apply("LeaderboardTeamFixedWindows",
      Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
          // We will get early (speculative) results as well as cumulative
          // processing of late data.
          .triggering(AfterWatermark.pastEndOfWindow()
              .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
                  .plusDelayOf(FIVE_MINUTES))
              .withLateFirings(AfterProcessingTime.pastFirstElementInPane()
                  .plusDelayOf(TEN_MINUTES)))
          .withAllowedLateness(allowedLateness)
          .accumulatingFiredPanes())
      // Extract and sum teamname/score pairs from the event data.
      .apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testProcessingTimeTrigger() {
  TestStream<Long> source = TestStream.create(VarLongCoder.of())
      .addElements(TimestampedValue.of(1L, new Instant(1000L)),
          TimestampedValue.of(2L, new Instant(2000L)))
      .advanceProcessingTime(Duration.standardMinutes(12))
      .addElements(TimestampedValue.of(3L, new Instant(3000L)))
      .advanceProcessingTime(Duration.standardMinutes(6))
      .advanceWatermarkToInfinity();

  PCollection<Long> sum = p.apply(source)
      .apply(Window.<Long>configure().triggering(AfterWatermark.pastEndOfWindow()
          .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
              .plusDelayOf(Duration.standardMinutes(5)))).accumulatingFiredPanes()
          .withAllowedLateness(Duration.ZERO))
      .apply(Sum.longsGlobally());

  PAssert.that(sum).inEarlyGlobalWindowPanes().containsInAnyOrder(3L, 6L);

  p.run();
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Test
@Category({ValidatesRunner.class, UsesTestStream.class})
public void testTriggeredDistinct() {
  Instant base = new Instant(0);
  TestStream<String> values = TestStream.create(StringUtf8Coder.of())
      .advanceWatermarkTo(base)
      .addElements(
          TimestampedValue.of("k1", base),
          TimestampedValue.of("k2", base.plus(Duration.standardSeconds(10))),
          TimestampedValue.of("k3", base.plus(Duration.standardSeconds(20))))
      .advanceProcessingTime(Duration.standardMinutes(1))
      .addElements(
          TimestampedValue.of("k1", base.plus(Duration.standardSeconds(30))),
          TimestampedValue.of("k2", base.plus(Duration.standardSeconds(40))),
          TimestampedValue.of("k3", base.plus(Duration.standardSeconds(50))))
      .advanceWatermarkToInfinity();

  PCollection<String> distinctValues = triggeredDistinctPipeline
      .apply(values)
      .apply(Window.<String>into(FixedWindows.of(Duration.standardMinutes(1)))
          .triggering(Repeatedly.forever(
              AfterProcessingTime.pastFirstElementInPane().plusDelayOf(
                  Duration.standardSeconds(30))))
          .withAllowedLateness(Duration.ZERO)
          .accumulatingFiredPanes())
      .apply(Distinct.<String>create());
  PAssert.that(distinctValues).containsInAnyOrder("k1", "k2", "k3");
  triggeredDistinctPipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Test
@Category({ValidatesRunner.class, UsesTestStream.class})
public void testTriggeredDistinctRepresentativeValues() {
  Instant base = new Instant(0);
  TestStream<KV<Integer, String>> values = TestStream.create(
      KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of()))
      .advanceWatermarkTo(base)
      .addElements(
          TimestampedValue.of(KV.of(1, "k1"), base),
          TimestampedValue.of(KV.of(2, "k2"), base.plus(Duration.standardSeconds(10))),
          TimestampedValue.of(KV.of(3, "k3"), base.plus(Duration.standardSeconds(20))))
      .advanceProcessingTime(Duration.standardMinutes(1))
      .addElements(
          TimestampedValue.of(KV.of(1, "k1"), base.plus(Duration.standardSeconds(30))),
          TimestampedValue.of(KV.of(2, "k2"), base.plus(Duration.standardSeconds(40))),
          TimestampedValue.of(KV.of(3, "k3"), base.plus(Duration.standardSeconds(50))))
      .advanceWatermarkToInfinity();

  PCollection<KV<Integer, String>> distinctValues = triggeredDistinctRepresentativePipeline
      .apply(values)
      .apply(Window.<KV<Integer, String>>into(FixedWindows.of(Duration.standardMinutes(1)))
          .triggering(Repeatedly.forever(
              AfterProcessingTime.pastFirstElementInPane().plusDelayOf(
                  Duration.standardSeconds(30))))
          .withAllowedLateness(Duration.ZERO)
          .accumulatingFiredPanes())
      .apply(Distinct.withRepresentativeValueFn(new Keys<Integer>())
          .withRepresentativeType(TypeDescriptor.of(Integer.class)));


  PAssert.that(distinctValues).containsInAnyOrder(
      KV.of(1, "k1"), KV.of(2, "k2"), KV.of(3, "k3"));
  triggeredDistinctRepresentativePipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Tests that when a processing time timers comes in after a window is expired it does not cause a
 * spurious output.
 */
@Test
@Category({ValidatesRunner.class, UsesTestStream.class})
public void testCombiningAccumulatingProcessingTime() throws Exception {
  PCollection<Integer> triggeredSums =
      p.apply(
              TestStream.create(VarIntCoder.of())
                  .advanceWatermarkTo(new Instant(0))
                  .addElements(
                      TimestampedValue.of(2, new Instant(2)),
                      TimestampedValue.of(5, new Instant(5)))
                  .advanceWatermarkTo(new Instant(100))
                  .advanceProcessingTime(Duration.millis(10))
                  .advanceWatermarkToInfinity())
          .apply(
              Window.<Integer>into(FixedWindows.of(Duration.millis(100)))
                  .withTimestampCombiner(TimestampCombiner.EARLIEST)
                  .accumulatingFiredPanes()
                  .withAllowedLateness(Duration.ZERO)
                  .triggering(
                      Repeatedly.forever(
                          AfterProcessingTime.pastFirstElementInPane()
                              .plusDelayOf(Duration.millis(10)))))
          .apply(Sum.integersGlobally().withoutDefaults());

  PAssert.that(triggeredSums)
      .containsInAnyOrder(7);

  p.run();
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Override
public PDone expand(PCollection<PubsubMessage> input) {
  input
      .apply(
          "PubsubUnboundedSink.Window",
          Window.<PubsubMessage>into(new GlobalWindows())
              .triggering(
                  Repeatedly.forever(
                      AfterFirst.of(
                          AfterPane.elementCountAtLeast(publishBatchSize),
                          AfterProcessingTime.pastFirstElementInPane().plusDelayOf(maxLatency))))
              .discardingFiredPanes())
      .apply("PubsubUnboundedSink.Shard", ParDo.of(new ShardFn(numShards, recordIdMethod)))
      .setCoder(KvCoder.of(VarIntCoder.of(), CODER))
      .apply(GroupByKey.<Integer, OutgoingMessage>create())
      .apply(
          "PubsubUnboundedSink.Writer",
          ParDo.of(
              new WriterFn(
                  pubsubFactory,
                  topic,
                  timestampAttribute,
                  idAttribute,
                  publishBatchSize,
                  publishBatchBytes)));
  return PDone.in(input.getPipeline());
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
private RunnerApi.Trigger convertSpecific(AfterProcessingTime v) {
  RunnerApi.Trigger.AfterProcessingTime.Builder builder =
      RunnerApi.Trigger.AfterProcessingTime.newBuilder();

  for (TimestampTransform transform : v.getTimestampTransforms()) {
    builder.addTimestampTransforms(convertTimestampTransform(transform));
  }

  return RunnerApi.Trigger.newBuilder().setAfterProcessingTime(builder).build();
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
@Parameters(name = "{index}: {0}")
public static Iterable<PCollection<?>> data() {
  Pipeline pipeline = TestPipeline.create();
  PCollection<Integer> ints = pipeline.apply("ints", Create.of(1, 2, 3));
  PCollection<Long> longs = pipeline.apply("unbounded longs", GenerateSequence.from(0));
  PCollection<Long> windowedLongs =
      longs.apply(
          "into fixed windows",
          Window.<Long>into(FixedWindows.of(Duration.standardMinutes(10L))));
  PCollection<KV<String, Iterable<String>>> groupedStrings =
      pipeline
          .apply(
              "kvs", Create.of(KV.of("foo", "spam"), KV.of("bar", "ham"), KV.of("baz", "eggs")))
          .apply("group", GroupByKey.<String, String>create());
  PCollection<Long> coderLongs =
      pipeline
          .apply("counts with alternative coder", GenerateSequence.from(0).to(10))
          .setCoder(BigEndianLongCoder.of());
  PCollection<Integer> allCustomInts =
      pipeline
          .apply(
              "intsWithCustomCoder",
              Create.of(1, 2)
                  .withCoder(new AutoValue_PCollectionTranslationTest_CustomIntCoder()))
          .apply(
              "into custom windows",
              Window.<Integer>into(new CustomWindows())
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(
                              AfterFirst.of(
                                  AfterPane.elementCountAtLeast(5),
                                  AfterProcessingTime.pastFirstElementInPane()
                                      .plusDelayOf(Duration.millis(227L)))))
                  .accumulatingFiredPanes()
                  .withAllowedLateness(Duration.standardMinutes(12L)));
  return ImmutableList.<PCollection<?>>of(ints, longs, windowedLongs, coderLongs, groupedStrings);
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Tests that a processing time timer does not cause window GC.
 */
@Test
public void testProcessingTimeTimerDoesNotGc() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withTrigger(
              Repeatedly.forever(
                  AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceProcessingTime(new Instant(5000));
  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceProcessingTime(new Instant(10000));

  tester.assertHasOnlyGlobalAndStateFor(
      new IntervalWindow(new Instant(0), new Instant(100)));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0))));
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Tests that when a processing time timer comes in after a window is expired
 * it is just ignored.
 */
@Test
public void testLateProcessingTimeTimer() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withTrigger(
              Repeatedly.forever(
                  AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceProcessingTime(new Instant(5000));
  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  // After this advancement, the window is expired and only the GC process
  // should be allowed to touch it
  tester.advanceInputWatermarkNoTimers(new Instant(100));

  // This should not output
  tester.advanceProcessingTime(new Instant(6000));

  assertThat(tester.extractOutput(), emptyIterable());
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Tests that when a processing time timer comes in after a window is expired
 * but in the same bundle it does not cause a spurious output.
 */
@Test
public void testCombiningAccumulatingProcessingTime() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withTrigger(
              Repeatedly.forever(
                  AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceProcessingTime(new Instant(5000));
  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceInputWatermarkNoTimers(new Instant(100));
  tester.advanceProcessingTimeNoTimers(new Instant(5010));

  // Fires the GC/EOW timer at the same time as the processing time timer.
  tester.fireTimers(
      new IntervalWindow(new Instant(0), new Instant(100)),
      TimestampedValue.of(TimeDomain.EVENT_TIME, new Instant(100)),
      TimestampedValue.of(TimeDomain.PROCESSING_TIME, new Instant(5010)));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Tests that when a processing time timers comes in after a window is expired
 * and GC'd it does not cause a spurious output.
 */
@Test
public void testCombiningAccumulatingProcessingTimeSeparateBundles() throws Exception {
  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100)))
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES)
          .withAllowedLateness(Duration.ZERO)
          .withTrigger(
              Repeatedly.forever(
                  AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.advanceProcessingTime(new Instant(5000));
  injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100
  injectElement(tester, 5);

  tester.advanceInputWatermark(new Instant(100));
  tester.advanceProcessingTime(new Instant(5011));

  assertThat(
      tester.extractOutput(),
      contains(
          isSingleWindowedValue(
              equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * We should fire an empty ON_TIME pane in the GlobalWindow when the watermark moves to
 * end-of-time.
 */
@Test
public void fireEmptyOnDrainInGlobalWindowIfRequested() throws Exception {
  ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester =
      ReduceFnTester.nonCombining(
          WindowingStrategy.of(new GlobalWindows())
                           .withTrigger(Repeatedly.<GlobalWindow>forever(
                               AfterProcessingTime.pastFirstElementInPane().plusDelayOf(
                                   new Duration(3))))
                           .withMode(AccumulationMode.DISCARDING_FIRED_PANES));

  final int n = 20;
  for (int i = 0; i < n; i++) {
    tester.advanceProcessingTime(new Instant(i));
    tester.injectElements(TimestampedValue.of(i, new Instant(i)));
  }
  tester.advanceProcessingTime(new Instant(n + 4));
  List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
  assertEquals((n + 3) / 4, output.size());
  for (int i = 0; i < output.size(); i++) {
    assertEquals(Timing.EARLY, output.get(i).getPane().getTiming());
    assertEquals(i, output.get(i).getPane().getIndex());
    assertEquals(4, Iterables.size(output.get(i).getValue()));
  }

  tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);

  output = tester.extractOutput();
  assertEquals(1, output.size());
  assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming());
  assertEquals((n + 3) / 4, output.get(0).getPane().getIndex());
  assertEquals(0, Iterables.size(output.get(0).getValue()));
}
 

import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime; //导入依赖的package包/类
/**
 * Get the beam pipeline
 * @return the pipeline
 */
public Pipeline getPipeline() {

    LOGGER.debug("Building Beam Pipeline");
    final PipelineOptions options = PipelineOptionsFactory.create();
    final Pipeline p = Pipeline.create(options);

    // Add membership triples
    p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_LDP_MEMBERSHIP_ADD).withoutMetadata())
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferMembership.getIRIString(), true)))
        .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE));

    // Delete membership triples
    p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_LDP_MEMBERSHIP_DELETE).withoutMetadata())
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferMembership.getIRIString(), false)))
        .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE));

    // Add containment triples
    p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_LDP_CONTAINMENT_ADD).withoutMetadata())
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), true)))
        .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE));

    // Delete containment triples
    p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_LDP_CONTAINMENT_DELETE).withoutMetadata())
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), false)))
        .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE));

    if (aggregateSeconds > 0) {
        // Aggregate cache writes
        p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE).withoutMetadata())
            .apply(Window.<KV<String, String>>into(FixedWindows.of(Duration.standardSeconds(aggregateSeconds)))
                    .triggering(AfterProcessingTime.pastFirstElementInPane()
                            .plusDelayOf(Duration.standardSeconds(aggregateSeconds)))
                    .discardingFiredPanes().withAllowedLateness(Duration.ZERO))
            .apply(Combine.perKey(x -> x.iterator().next()))
            .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE));
    } else {
        // Skip aggregation
        p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_CACHE_AGGREGATE).withoutMetadata())
            .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_CACHE));
    }

    // Write to cache and dispatch to the event bus
    p.apply(getKafkaReader(bootstrapServers).withTopic(TOPIC_CACHE).withoutMetadata())
        .apply(ParDo.of(new CacheWriter(dataConfiguration)))
        .apply(ParDo.of(new EventProcessor(baseUrlConfiguration)))
        .apply(getKafkaWriter(bootstrapServers).withTopic(TOPIC_EVENT));

    return p;
}