Java FixedWindows.of方法代码示例

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testElementsAtAlmostPositiveInfinity() {
  Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
  TestStream<String> stream =
      TestStream.create(StringUtf8Coder.of())
          .addElements(
              TimestampedValue.of("foo", endOfGlobalWindow),
              TimestampedValue.of("bar", endOfGlobalWindow))
          .advanceWatermarkToInfinity();

  FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
  PCollection<String> windowedValues =
      p.apply(stream)
          .apply(Window.<String>into(windows))
          .apply(WithKeys.<Integer, String>of(1))
          .apply(GroupByKey.<Integer, String>create())
          .apply(Values.<Iterable<String>>create())
          .apply(Flatten.<String>iterables());

  PAssert.that(windowedValues)
      .inWindow(windows.assignWindow(endOfGlobalWindow))
      .containsInAnyOrder("foo", "bar");
  p.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(-100));
  assertThat(
      window.maxTimestamp().plus(allowedLateness),
      Matchers.<ReadableInstant>greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void testElementsAtAlmostPositiveInfinity() throws IOException {
  Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
  CreateStream<String> source =
      CreateStream.of(StringUtf8Coder.of(), batchDuration())
          .nextBatch(
              TimestampedValue.of("foo", endOfGlobalWindow),
              TimestampedValue.of("bar", endOfGlobalWindow))
          .advanceNextBatchWatermarkToInfinity();

  FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
  PCollection<String> windowedValues = p.apply(source)
      .apply(Window.<String>into(windows))
      .apply(WithKeys.<Integer, String>of(1))
      .apply(GroupByKey.<Integer, String>create())
      .apply(Values.<Iterable<String>>create())
      .apply(Flatten.<String>iterables());

  PAssert.that(windowedValues)
      .inWindow(windows.assignWindow(GlobalWindow.INSTANCE.maxTimestamp()))
      .containsInAnyOrder("foo", "bar");
  p.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void beforeEndOfGlobalWindowSame() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  Duration allowedLateness = Duration.standardMinutes(2);
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn)
          .withAllowedLateness(allowedLateness);

  IntervalWindow window = windowFn.assignWindow(new Instant(10));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(window.maxTimestamp().plus(allowedLateness)));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindow() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
  WindowingStrategy<?, ?> strategy =
      WindowingStrategy.globalDefault()
          .withWindowFn(windowFn);

  IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
  assertThat(
      window.maxTimestamp(),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
  assertThat(
      LateDataUtils.garbageCollectionTime(window, strategy),
      equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void multipleCallbacksShouldFireFires() throws Exception {
  CountDownLatch latch = new CountDownLatch(2);
  WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
  IntervalWindow window =
      new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
  executor.callOnGuaranteedFiring(
      create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));
  executor.callOnGuaranteedFiring(
      create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));

  executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(10)));
  assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void noCallbacksShouldFire() throws Exception {
  CountDownLatch latch = new CountDownLatch(1);
  WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
  IntervalWindow window =
      new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
  executor.callOnGuaranteedFiring(
      create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));

  executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(5)));
  assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void unrelatedStepShouldNotFire() throws Exception {
  CountDownLatch latch = new CountDownLatch(1);
  WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
  IntervalWindow window =
      new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
  executor.callOnGuaranteedFiring(
      sum, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));

  executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(20)));
  assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testWindowedCombineGloballyAsSingletonView() {
  FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(1));
  final PCollectionView<Integer> view =
      pipeline
          .apply(
              "CreateSideInput",
              Create.timestamped(
                  TimestampedValue.of(1, new Instant(100)),
                  TimestampedValue.of(3, new Instant(100))))
          .apply("WindowSideInput", Window.<Integer>into(windowFn))
          .apply("CombineSideInput", Sum.integersGlobally().asSingletonView());

  TimestampedValue<Void> nonEmptyElement = TimestampedValue.of(null, new Instant(100));
  TimestampedValue<Void> emptyElement = TimestampedValue.atMinimumTimestamp(null);
  PCollection<Integer> output =
      pipeline
          .apply(
              "CreateMainInput",
              Create.timestamped(nonEmptyElement, emptyElement).withCoder(VoidCoder.of()))
          .apply("WindowMainInput", Window.<Void>into(windowFn))
          .apply(
              "OutputSideInput",
              ParDo.of(
                      new DoFn<Void, Integer>() {
                        @ProcessElement
                        public void processElement(ProcessContext c) {
                          c.output(c.sideInput(view));
                        }
                      })
                  .withSideInputs(view));

  PAssert.that(output).containsInAnyOrder(4, 0);
  PAssert.that(output)
      .inWindow(windowFn.assignWindow(nonEmptyElement.getTimestamp()))
      .containsInAnyOrder(4);
  PAssert.that(output)
      .inWindow(windowFn.assignWindow(emptyElement.getTimestamp()))
      .containsInAnyOrder(0);
  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testDiscardingMode() {
  TestStream<String> stream =
      TestStream.create(StringUtf8Coder.of())
          .advanceWatermarkTo(new Instant(0))
          .addElements(
              TimestampedValue.of("firstPane", new Instant(100)),
              TimestampedValue.of("alsoFirstPane", new Instant(200)))
          .addElements(TimestampedValue.of("onTimePane", new Instant(500)))
          .advanceWatermarkTo(new Instant(1001L))
          .addElements(
              TimestampedValue.of("finalLatePane", new Instant(750)),
              TimestampedValue.of("alsoFinalLatePane", new Instant(250)))
          .advanceWatermarkToInfinity();

  FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
  Duration allowedLateness = Duration.millis(5000L);
  PCollection<String> values =
      p.apply(stream)
          .apply(
              Window.<String>into(windowFn)
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(2))
                          .withLateFirings(Never.ever()))
                  .discardingFiredPanes()
                  .withAllowedLateness(allowedLateness))
          .apply(WithKeys.<Integer, String>of(1))
          .apply(GroupByKey.<Integer, String>create())
          .apply(Values.<Iterable<String>>create())
          .apply(Flatten.<String>iterables());

  IntervalWindow window = windowFn.assignWindow(new Instant(100));
  PAssert.that(values)
      .inWindow(window)
      .containsInAnyOrder(
          "firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
  PAssert.that(values)
      .inCombinedNonLatePanes(window)
      .containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
  PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
  PAssert.that(values)
      .inFinalPane(window)
      .containsInAnyOrder("finalLatePane", "alsoFinalLatePane");

  p.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
/**
 * Tests that the garbage collection time for a fixed window does not overflow the end of time.
 */
@Test
public void testFixedWindowEndOfTimeGarbageCollection() throws Exception {

  Duration allowedLateness = Duration.standardDays(365);
  Duration windowSize = Duration.millis(10);
  WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(windowSize);

  // This timestamp falls into a window where the end of the window is before the end of the
  // global window - the "end of time" - yet its expiration time is after.
  final Instant elementTimestamp =
      GlobalWindow.INSTANCE.maxTimestamp().minus(allowedLateness).plus(1);

  IntervalWindow window = Iterables.getOnlyElement(
      windowFn.assignWindows(
          windowFn.new AssignContext() {
            @Override
            public Object element() {
              throw new UnsupportedOperationException();
            }
            @Override
            public Instant timestamp() {
              return elementTimestamp;
            }

            @Override
            public BoundedWindow window() {
              throw new UnsupportedOperationException();
            }
          }));

  assertTrue(
      window.maxTimestamp().isBefore(GlobalWindow.INSTANCE.maxTimestamp()));
  assertTrue(
      window.maxTimestamp().plus(allowedLateness).isAfter(GlobalWindow.INSTANCE.maxTimestamp()));

  // Test basic execution of a trigger using a non-combining window set and accumulating mode.

  WindowingStrategy<?, IntervalWindow> strategy =
      WindowingStrategy.of((WindowFn<?, IntervalWindow>) windowFn)
          .withTimestampCombiner(TimestampCombiner.EARLIEST)
          .withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(Never.ever()))
          .withMode(AccumulationMode.DISCARDING_FIRED_PANES)
          .withAllowedLateness(allowedLateness);

  ReduceFnTester<Integer, Integer, IntervalWindow> tester =
      ReduceFnTester
          .combining(strategy, Sum.ofIntegers(), VarIntCoder.of());

  tester.injectElements(TimestampedValue.of(13, elementTimestamp));

  // Should fire ON_TIME pane and there will be a checkState that the cleanup time
  // is prior to timestamp max value
  tester.advanceInputWatermark(window.maxTimestamp());

  // Nothing in the ON_TIME pane (not governed by triggers, but by ReduceFnRunner)
  assertThat(tester.extractOutput(), emptyIterable());

  tester.injectElements(TimestampedValue.of(42, elementTimestamp));

  // Now the final pane should fire, demonstrating that the GC time was truncated
  tester.advanceInputWatermark(GlobalWindow.INSTANCE.maxTimestamp());
  assertThat(tester.extractOutput(), contains(isWindowedValue(equalTo(55))));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void testDiscardingMode() throws IOException {
  CreateStream<String> source =
      CreateStream.of(StringUtf8Coder.of(), batchDuration())
                  .nextBatch(
                      TimestampedValue.of("firstPane", new Instant(100)),
                      TimestampedValue.of("alsoFirstPane", new Instant(200)))
                  .advanceWatermarkForNextBatch(new Instant(1001L))
                  .nextBatch(
                      TimestampedValue.of("onTimePane", new Instant(500)))
                  .advanceNextBatchWatermarkToInfinity()
                  .nextBatch(
                      TimestampedValue.of("finalLatePane", new Instant(750)),
                      TimestampedValue.of("alsoFinalLatePane", new Instant(250)));

  FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
  Duration allowedLateness = Duration.millis(5000L);
  PCollection<String> values =
      p.apply(source)
          .apply(
              Window.<String>into(windowFn)
                  .triggering(
                      AfterWatermark.pastEndOfWindow()
                          .withEarlyFirings(AfterPane.elementCountAtLeast(2))
                          .withLateFirings(Never.ever()))
                  .discardingFiredPanes()
                  .withAllowedLateness(allowedLateness))
          .apply(WithKeys.<Integer, String>of(1))
          .apply(GroupByKey.<Integer, String>create())
          .apply(Values.<Iterable<String>>create())
          .apply(Flatten.<String>iterables());

  IntervalWindow window = windowFn.assignWindow(new Instant(100));
  PAssert.that(values)
      .inWindow(window)
      .containsInAnyOrder(
          "firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
  PAssert.that(values)
      .inCombinedNonLatePanes(window)
      .containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
  PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
  PAssert.that(values)
      .inFinalPane(window)
      .containsInAnyOrder("finalLatePane", "alsoFinalLatePane");

  p.run();
}