本文整理汇总了Java中org.apache.beam.sdk.transforms.windowing.PaneInfo类的典型用法代码示例。如果您正苦于以下问题:Java PaneInfo类的具体用法?Java PaneInfo怎么用?Java PaneInfo使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
PaneInfo类属于org.apache.beam.sdk.transforms.windowing包,在下文中一共展示了PaneInfo类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testPaneInfoFinalAndOnTime
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void testPaneInfoFinalAndOnTime() throws Exception {
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(
WindowingStrategy.of(FixedWindows.of(Duration.millis(10)))
.withTrigger(
Repeatedly.forever(AfterPane.elementCountAtLeast(2))
.orFinally(AfterWatermark.pastEndOfWindow()))
.withMode(AccumulationMode.DISCARDING_FIRED_PANES)
.withAllowedLateness(Duration.millis(100))
.withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
tester.advanceInputWatermark(new Instant(0));
// Should trigger due to element count
tester.injectElements(
TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
assertThat(
tester.extractOutput(),
contains(WindowMatchers.valueWithPaneInfo(
PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));
tester.advanceInputWatermark(new Instant(150));
assertThat(tester.extractOutput(), contains(
WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0))));
}
示例2: emitElement
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
/**
* Emit the current element from the given Reader. The reader is guaranteed to have data.
*/
private void emitElement(
SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx,
UnboundedSource.UnboundedReader<OutputT> reader) {
// make sure that reader state update and element emission are atomic
// with respect to snapshots
synchronized (ctx.getCheckpointLock()) {
OutputT item = reader.getCurrent();
byte[] recordId = reader.getCurrentRecordId();
Instant timestamp = reader.getCurrentTimestamp();
WindowedValue<ValueWithRecordId<OutputT>> windowedValue =
WindowedValue.of(new ValueWithRecordId<>(item, recordId), timestamp,
GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
ctx.collectWithTimestamp(windowedValue, timestamp.getMillis());
}
}
示例3: of
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
/**
* Returns a {@code WindowedValue} with the given value, timestamp,
* and windows.
*/
public static <T> WindowedValue<T> of(
T value,
Instant timestamp,
Collection<? extends BoundedWindow> windows,
PaneInfo pane) {
checkNotNull(pane);
if (windows.size() == 0 && BoundedWindow.TIMESTAMP_MIN_VALUE.equals(timestamp)) {
return valueInEmptyWindows(value, pane);
} else if (windows.size() == 1) {
return of(value, timestamp, windows.iterator().next(), pane);
} else {
return new TimestampedValueInMultipleWindows<>(value, timestamp, windows, pane);
}
}
示例4: processElementSideInputNotReady
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void processElementSideInputNotReady() {
when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class)))
.thenReturn(false);
SimplePushbackSideInputDoFnRunner<Integer, Integer> runner =
createRunner(ImmutableList.<PCollectionView<?>>of(singletonView));
WindowedValue<Integer> oneWindow =
WindowedValue.of(
2,
new Instant(-2),
new IntervalWindow(new Instant(-500L), new Instant(0L)),
PaneInfo.ON_TIME_AND_ONLY_FIRING);
Iterable<WindowedValue<Integer>> oneWindowPushback =
runner.processElementInReadyWindows(oneWindow);
assertThat(oneWindowPushback, containsInAnyOrder(oneWindow));
assertThat(underlying.inputElems, Matchers.<WindowedValue<Integer>>emptyIterable());
}
示例5: noMutationKeyedBundleSucceeds
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void noMutationKeyedBundleSucceeds() {
UncommittedBundle<byte[]> keyed =
factory.createKeyedBundle(StructuralKey.of("mykey", StringUtf8Coder.of()), transformed);
WindowedValue<byte[]> windowedArray =
WindowedValue.of(
new byte[] {4, 8, 12},
new Instant(891L),
new IntervalWindow(new Instant(0), new Instant(1000)),
PaneInfo.ON_TIME_AND_ONLY_FIRING);
keyed.add(windowedArray);
CommittedBundle<byte[]> committed = keyed.commit(Instant.now());
assertThat(committed.getElements(), containsInAnyOrder(windowedArray));
}
示例6: mutationBeforeAddCreateBundleSucceeds
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void mutationBeforeAddCreateBundleSucceeds() {
UncommittedBundle<byte[]> intermediate = factory.createBundle(transformed);
byte[] array = new byte[] {4, 8, 12};
WindowedValue<byte[]> windowedArray =
WindowedValue.of(
array,
new Instant(891L),
new IntervalWindow(new Instant(0), new Instant(1000)),
PaneInfo.ON_TIME_AND_ONLY_FIRING);
array[2] = -3;
intermediate.add(windowedArray);
CommittedBundle<byte[]> committed = intermediate.commit(Instant.now());
assertThat(committed.getElements(), containsInAnyOrder(windowedArray));
}
示例7: windowedFilename
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Override
public ResourceId windowedFilename(
int shardNumber,
int numShards,
BoundedWindow window,
PaneInfo paneInfo,
OutputFileHints outputFileHints) {
String paneStr = paneInfoToString(paneInfo);
String windowStr = windowToString(window);
return constructName(
params.baseFilename.get(),
params.shardTemplate,
params.suffix + outputFileHints.getSuggestedFilenameSuffix(),
shardNumber,
numShards,
paneStr,
windowStr);
}
示例8: testFixedWindowsEowAndGcTogether
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
/**
* When the watermark passes the end-of-window and window expiration time
* in a single update, this tests that it does not crash.
*/
@Test
public void testFixedWindowsEowAndGcTogether() throws Exception {
ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester =
ReduceFnTester.nonCombining(
FixedWindows.of(Duration.millis(10)),
DefaultTriggerStateMachine.<IntervalWindow>of(),
AccumulationMode.ACCUMULATING_FIRED_PANES,
Duration.millis(50),
ClosingBehavior.FIRE_ALWAYS);
tester.setAutoAdvanceOutputWatermark(true);
tester.advanceInputWatermark(new Instant(0));
injectElement(tester, 1);
tester.advanceInputWatermark(new Instant(100));
assertThat(
tester.extractOutput(),
contains(
isSingleWindowedValue(
contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME))));
}
示例9: call
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public WindowedValue<T> call(WindowedValue<T> windowedValue) throws Exception {
final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows());
final T element = windowedValue.getValue();
final Instant timestamp = windowedValue.getTimestamp();
Collection<W> windows =
((WindowFn<T, W>) fn).assignWindows(
((WindowFn<T, W>) fn).new AssignContext() {
@Override
public T element() {
return element;
}
@Override
public Instant timestamp() {
return timestamp;
}
@Override
public BoundedWindow window() {
return boundedWindow;
}
});
return WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING);
}
示例10: onTimePane
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void onTimePane() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
PaneExtractors.onTimePane();
Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
ImmutableList.of(
ValueInSingleWindow.of(
4,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)),
ValueInSingleWindow.of(
2,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)));
assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(2, 4));
}
示例11: finalPane
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void finalPane() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
PaneExtractors.finalPane();
Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
ImmutableList.of(
ValueInSingleWindow.of(
8,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, true, Timing.LATE, 2L, 1L)),
ValueInSingleWindow.of(
4,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)),
ValueInSingleWindow.of(
1,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(true, false, Timing.EARLY)));
assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(8));
}
示例12: testIsWindowedValueExact
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void testIsWindowedValueExact() {
long timestamp = 100;
long windowStart = 0;
long windowEnd = 200;
assertThat(
WindowedValue.of(
"hello",
new Instant(timestamp),
new IntervalWindow(new Instant(windowStart), new Instant(windowEnd)),
PaneInfo.NO_FIRING),
WindowMatchers.isWindowedValue(
"hello",
new Instant(timestamp),
ImmutableList.of(new IntervalWindow(new Instant(windowStart), new Instant(windowEnd))),
PaneInfo.NO_FIRING));
}
示例13: writeForElementInMultipleWindowsSucceeds
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void writeForElementInMultipleWindowsSucceeds() throws Exception {
ImmutableList.Builder<WindowedValue<?>> valuesBuilder = ImmutableList.builder();
for (Object materializedValue : materializeValuesFor(View.asSingleton(), 2.875)) {
valuesBuilder.add(WindowedValue.of(
materializedValue,
FIRST_WINDOW.maxTimestamp().minus(200L),
ImmutableList.of(FIRST_WINDOW, SECOND_WINDOW),
PaneInfo.ON_TIME_AND_ONLY_FIRING));
}
container.write(singletonView, valuesBuilder.build());
assertThat(
container
.createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
.get(singletonView, FIRST_WINDOW),
equalTo(2.875));
assertThat(
container
.createReaderForViews(ImmutableList.<PCollectionView<?>>of(singletonView))
.get(singletonView, SECOND_WINDOW),
equalTo(2.875));
}
示例14: allPanesMultiplePanes
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Test
public void allPanesMultiplePanes() {
SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor =
PaneExtractors.allPanes();
Iterable<ValueInSingleWindow<Integer>> onlyOnTime =
ImmutableList.of(
ValueInSingleWindow.of(
8,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.LATE, 2L, 1L)),
ValueInSingleWindow.of(
4,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)),
ValueInSingleWindow.of(
1,
new Instant(0L),
GlobalWindow.INSTANCE,
PaneInfo.createPane(true, false, Timing.EARLY)));
assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8, 1));
}
示例15: windowedFilename
import org.apache.beam.sdk.transforms.windowing.PaneInfo; //导入依赖的package包/类
@Override
public ResourceId windowedFilename(
int shardNumber,
int numShards,
BoundedWindow window,
PaneInfo paneInfo,
OutputFileHints outputFileHints) {
DecimalFormat df = new DecimalFormat("0000");
IntervalWindow intervalWindow = (IntervalWindow) window;
String filename =
String.format(
"%s-%s-of-%s%s%s",
filenamePrefixForWindow(intervalWindow),
df.format(shardNumber),
df.format(numShards),
outputFileHints.getSuggestedFilenameSuffix(),
suffix);
return baseFilename
.getCurrentDirectory()
.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
}