本文整理汇总了Java中org.apache.beam.sdk.transforms.windowing.FixedWindows.of方法的典型用法代码示例。如果您正苦于以下问题:Java FixedWindows.of方法的具体用法?Java FixedWindows.of怎么用?Java FixedWindows.of使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.beam.sdk.transforms.windowing.FixedWindows
的用法示例。
在下文中一共展示了FixedWindows.of方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testElementsAtAlmostPositiveInfinity
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testElementsAtAlmostPositiveInfinity() {
Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
TestStream<String> stream =
TestStream.create(StringUtf8Coder.of())
.addElements(
TimestampedValue.of("foo", endOfGlobalWindow),
TimestampedValue.of("bar", endOfGlobalWindow))
.advanceWatermarkToInfinity();
FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
PCollection<String> windowedValues =
p.apply(stream)
.apply(Window.<String>into(windows))
.apply(WithKeys.<Integer, String>of(1))
.apply(GroupByKey.<Integer, String>create())
.apply(Values.<Iterable<String>>create())
.apply(Flatten.<String>iterables());
PAssert.that(windowedValues)
.inWindow(windows.assignWindow(endOfGlobalWindow))
.containsInAnyOrder("foo", "bar");
p.run();
}
示例2: garbageCollectionTimeAfterEndOfGlobalWindowWithLateness
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
WindowingStrategy<?, ?> strategy =
WindowingStrategy.globalDefault()
.withWindowFn(windowFn)
.withAllowedLateness(allowedLateness);
IntervalWindow window = windowFn.assignWindow(new Instant(-100));
assertThat(
window.maxTimestamp().plus(allowedLateness),
Matchers.<ReadableInstant>greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
assertThat(
LateDataUtils.garbageCollectionTime(window, strategy),
equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
示例3: testElementsAtAlmostPositiveInfinity
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void testElementsAtAlmostPositiveInfinity() throws IOException {
Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
CreateStream<String> source =
CreateStream.of(StringUtf8Coder.of(), batchDuration())
.nextBatch(
TimestampedValue.of("foo", endOfGlobalWindow),
TimestampedValue.of("bar", endOfGlobalWindow))
.advanceNextBatchWatermarkToInfinity();
FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
PCollection<String> windowedValues = p.apply(source)
.apply(Window.<String>into(windows))
.apply(WithKeys.<Integer, String>of(1))
.apply(GroupByKey.<Integer, String>create())
.apply(Values.<Iterable<String>>create())
.apply(Flatten.<String>iterables());
PAssert.that(windowedValues)
.inWindow(windows.assignWindow(GlobalWindow.INSTANCE.maxTimestamp()))
.containsInAnyOrder("foo", "bar");
p.run();
}
示例4: beforeEndOfGlobalWindowSame
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void beforeEndOfGlobalWindowSame() {
FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
Duration allowedLateness = Duration.standardMinutes(2);
WindowingStrategy<?, ?> strategy =
WindowingStrategy.globalDefault()
.withWindowFn(windowFn)
.withAllowedLateness(allowedLateness);
IntervalWindow window = windowFn.assignWindow(new Instant(10));
assertThat(
LateDataUtils.garbageCollectionTime(window, strategy),
equalTo(window.maxTimestamp().plus(allowedLateness)));
}
示例5: garbageCollectionTimeAfterEndOfGlobalWindow
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void garbageCollectionTimeAfterEndOfGlobalWindow() {
FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
WindowingStrategy<?, ?> strategy =
WindowingStrategy.globalDefault()
.withWindowFn(windowFn);
IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE));
assertThat(
window.maxTimestamp(),
equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
assertThat(
LateDataUtils.garbageCollectionTime(window, strategy),
equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}
示例6: multipleCallbacksShouldFireFires
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void multipleCallbacksShouldFireFires() throws Exception {
CountDownLatch latch = new CountDownLatch(2);
WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
IntervalWindow window =
new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
executor.callOnGuaranteedFiring(
create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));
executor.callOnGuaranteedFiring(
create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));
executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(10)));
assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true));
}
示例7: noCallbacksShouldFire
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void noCallbacksShouldFire() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
IntervalWindow window =
new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
executor.callOnGuaranteedFiring(
create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));
executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(5)));
assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false));
}
示例8: unrelatedStepShouldNotFire
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void unrelatedStepShouldNotFire() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10));
IntervalWindow window =
new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10)));
executor.callOnGuaranteedFiring(
sum, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch));
executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(20)));
assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false));
}
示例9: testWindowedCombineGloballyAsSingletonView
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testWindowedCombineGloballyAsSingletonView() {
FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(1));
final PCollectionView<Integer> view =
pipeline
.apply(
"CreateSideInput",
Create.timestamped(
TimestampedValue.of(1, new Instant(100)),
TimestampedValue.of(3, new Instant(100))))
.apply("WindowSideInput", Window.<Integer>into(windowFn))
.apply("CombineSideInput", Sum.integersGlobally().asSingletonView());
TimestampedValue<Void> nonEmptyElement = TimestampedValue.of(null, new Instant(100));
TimestampedValue<Void> emptyElement = TimestampedValue.atMinimumTimestamp(null);
PCollection<Integer> output =
pipeline
.apply(
"CreateMainInput",
Create.timestamped(nonEmptyElement, emptyElement).withCoder(VoidCoder.of()))
.apply("WindowMainInput", Window.<Void>into(windowFn))
.apply(
"OutputSideInput",
ParDo.of(
new DoFn<Void, Integer>() {
@ProcessElement
public void processElement(ProcessContext c) {
c.output(c.sideInput(view));
}
})
.withSideInputs(view));
PAssert.that(output).containsInAnyOrder(4, 0);
PAssert.that(output)
.inWindow(windowFn.assignWindow(nonEmptyElement.getTimestamp()))
.containsInAnyOrder(4);
PAssert.that(output)
.inWindow(windowFn.assignWindow(emptyElement.getTimestamp()))
.containsInAnyOrder(0);
pipeline.run();
}
示例10: testDiscardingMode
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testDiscardingMode() {
TestStream<String> stream =
TestStream.create(StringUtf8Coder.of())
.advanceWatermarkTo(new Instant(0))
.addElements(
TimestampedValue.of("firstPane", new Instant(100)),
TimestampedValue.of("alsoFirstPane", new Instant(200)))
.addElements(TimestampedValue.of("onTimePane", new Instant(500)))
.advanceWatermarkTo(new Instant(1001L))
.addElements(
TimestampedValue.of("finalLatePane", new Instant(750)),
TimestampedValue.of("alsoFinalLatePane", new Instant(250)))
.advanceWatermarkToInfinity();
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
PCollection<String> values =
p.apply(stream)
.apply(
Window.<String>into(windowFn)
.triggering(
AfterWatermark.pastEndOfWindow()
.withEarlyFirings(AfterPane.elementCountAtLeast(2))
.withLateFirings(Never.ever()))
.discardingFiredPanes()
.withAllowedLateness(allowedLateness))
.apply(WithKeys.<Integer, String>of(1))
.apply(GroupByKey.<Integer, String>create())
.apply(Values.<Iterable<String>>create())
.apply(Flatten.<String>iterables());
IntervalWindow window = windowFn.assignWindow(new Instant(100));
PAssert.that(values)
.inWindow(window)
.containsInAnyOrder(
"firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
PAssert.that(values)
.inCombinedNonLatePanes(window)
.containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
PAssert.that(values)
.inFinalPane(window)
.containsInAnyOrder("finalLatePane", "alsoFinalLatePane");
p.run();
}
示例11: testFixedWindowEndOfTimeGarbageCollection
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
/**
* Tests that the garbage collection time for a fixed window does not overflow the end of time.
*/
@Test
public void testFixedWindowEndOfTimeGarbageCollection() throws Exception {
Duration allowedLateness = Duration.standardDays(365);
Duration windowSize = Duration.millis(10);
WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(windowSize);
// This timestamp falls into a window where the end of the window is before the end of the
// global window - the "end of time" - yet its expiration time is after.
final Instant elementTimestamp =
GlobalWindow.INSTANCE.maxTimestamp().minus(allowedLateness).plus(1);
IntervalWindow window = Iterables.getOnlyElement(
windowFn.assignWindows(
windowFn.new AssignContext() {
@Override
public Object element() {
throw new UnsupportedOperationException();
}
@Override
public Instant timestamp() {
return elementTimestamp;
}
@Override
public BoundedWindow window() {
throw new UnsupportedOperationException();
}
}));
assertTrue(
window.maxTimestamp().isBefore(GlobalWindow.INSTANCE.maxTimestamp()));
assertTrue(
window.maxTimestamp().plus(allowedLateness).isAfter(GlobalWindow.INSTANCE.maxTimestamp()));
// Test basic execution of a trigger using a non-combining window set and accumulating mode.
WindowingStrategy<?, IntervalWindow> strategy =
WindowingStrategy.of((WindowFn<?, IntervalWindow>) windowFn)
.withTimestampCombiner(TimestampCombiner.EARLIEST)
.withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(Never.ever()))
.withMode(AccumulationMode.DISCARDING_FIRED_PANES)
.withAllowedLateness(allowedLateness);
ReduceFnTester<Integer, Integer, IntervalWindow> tester =
ReduceFnTester
.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
tester.injectElements(TimestampedValue.of(13, elementTimestamp));
// Should fire ON_TIME pane and there will be a checkState that the cleanup time
// is prior to timestamp max value
tester.advanceInputWatermark(window.maxTimestamp());
// Nothing in the ON_TIME pane (not governed by triggers, but by ReduceFnRunner)
assertThat(tester.extractOutput(), emptyIterable());
tester.injectElements(TimestampedValue.of(42, elementTimestamp));
// Now the final pane should fire, demonstrating that the GC time was truncated
tester.advanceInputWatermark(GlobalWindow.INSTANCE.maxTimestamp());
assertThat(tester.extractOutput(), contains(isWindowedValue(equalTo(55))));
}
示例12: testDiscardingMode
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入方法依赖的package包/类
@Test
public void testDiscardingMode() throws IOException {
CreateStream<String> source =
CreateStream.of(StringUtf8Coder.of(), batchDuration())
.nextBatch(
TimestampedValue.of("firstPane", new Instant(100)),
TimestampedValue.of("alsoFirstPane", new Instant(200)))
.advanceWatermarkForNextBatch(new Instant(1001L))
.nextBatch(
TimestampedValue.of("onTimePane", new Instant(500)))
.advanceNextBatchWatermarkToInfinity()
.nextBatch(
TimestampedValue.of("finalLatePane", new Instant(750)),
TimestampedValue.of("alsoFinalLatePane", new Instant(250)));
FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
Duration allowedLateness = Duration.millis(5000L);
PCollection<String> values =
p.apply(source)
.apply(
Window.<String>into(windowFn)
.triggering(
AfterWatermark.pastEndOfWindow()
.withEarlyFirings(AfterPane.elementCountAtLeast(2))
.withLateFirings(Never.ever()))
.discardingFiredPanes()
.withAllowedLateness(allowedLateness))
.apply(WithKeys.<Integer, String>of(1))
.apply(GroupByKey.<Integer, String>create())
.apply(Values.<Iterable<String>>create())
.apply(Flatten.<String>iterables());
IntervalWindow window = windowFn.assignWindow(new Instant(100));
PAssert.that(values)
.inWindow(window)
.containsInAnyOrder(
"firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
PAssert.that(values)
.inCombinedNonLatePanes(window)
.containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
PAssert.that(values)
.inFinalPane(window)
.containsInAnyOrder("finalLatePane", "alsoFinalLatePane");
p.run();
}