本文整理汇总了Java中org.apache.beam.sdk.transforms.windowing.FixedWindows类的典型用法代码示例。如果您正苦于以下问题:Java FixedWindows类的具体用法?Java FixedWindows怎么用?Java FixedWindows使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
FixedWindows类属于org.apache.beam.sdk.transforms.windowing包,在下文中一共展示了FixedWindows类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: main
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
Options options = PipelineOptionsFactory.fromArgs(args).withValidation()
.as(Options.class);
options.setRunner(FlinkRunner.class);
Pipeline p = Pipeline.create(options);
KafkaIO.Read<byte[], String> kafkaIOReader = KafkaIO.read()
.withBootstrapServers("192.168.99.100:32771")
.withTopics(Arrays.asList("beam".split(",")))
.updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest"))
.withValueCoder(StringUtf8Coder.of());
p.apply(kafkaIOReader.withoutMetadata())
.apply(Values.<String>create())
.apply(Window.<String>into(
FixedWindows.of(Duration.standardMinutes(options.getWindowSize()))))
.apply(new CountWords())
.apply(MapElements.via(new FormatAsTextFn()))
.apply("WriteCounts", TextIO.Write.to(options.getOutput()));
p.run();
}
示例2: main
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
/** Run a batch pipeline to calculate hourly team scores. */
public static void main(String[] args) throws Exception {
Options options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
Pipeline pipeline = Pipeline.create(options);
pipeline
.apply("ReadLogs", TextIO.read().from(options.getInput()))
.apply("SetTimestamps", WithTimestamps.of(new SetTimestampFn()))
.apply("FixedWindows", Window.<String>into(FixedWindows.of(ONE_HOUR)))
.apply("TeamScores", new CalculateTeamScores(options.getOutputPrefix()));
pipeline.run();
}
示例3: main
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
Options options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
Pipeline pipeline = Pipeline.create(options);
pipeline
.apply(KafkaIO.<String, String>read()
.withBootstrapServers(options.getKafkaBootstrapServer())
.withTopic(options.getTopic())
.withKeyDeserializer(StringDeserializer.class)
.withValueDeserializer(StringDeserializer.class)
.withTimestampFn(new SetTimestampFn()))
.apply("Values", ParDo.of(new ValuesFn()))
.apply("FixedWindows", Window.<String>into(FixedWindows.of(FIVE_MINUTES))
.triggering(AfterWatermark.pastEndOfWindow()
.withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
.plusDelayOf(TWO_MINUTES))
.withLateFirings(AfterPane.elementCountAtLeast(1)))
.withAllowedLateness(TEN_MINUTES)
.accumulatingFiredPanes())
.apply("TeamScore", new CalculateTeamScores(options.getOutputPrefix()));
pipeline.run();
}
示例4: expand
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
return infos.apply("LeaderboardTeamFixedWindows",
Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
// We will get early (speculative) results as well as cumulative
// processing of late data.
.triggering(AfterWatermark.pastEndOfWindow()
.withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
.plusDelayOf(FIVE_MINUTES))
.withLateFirings(AfterProcessingTime.pastFirstElementInPane()
.plusDelayOf(TEN_MINUTES)))
.withAllowedLateness(allowedLateness)
.accumulatingFiredPanes())
// Extract and sum teamname/score pairs from the event data.
.apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
示例5: testTotalFlow
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testTotalFlow () {
PCollection<KV<String, Integer>> flow = pipeline
.apply(Create.timestamped(TIME_STAMPED_INPUT))
.apply(ParDo.of(new ExtractFlowInfo()));
PCollection<TableRow> totalFlow = flow
.apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.standardMinutes(1))))
.apply(new TotalFlow("default"));
PCollection<String> results = totalFlow.apply(ParDo.of(new FormatResults()));
PAssert.that(results)
.containsInAnyOrder(canonicalFormat(OUT_ROW_1), canonicalFormat(OUT_ROW_2));
pipeline.run().waitUntilFinish();
}
示例6: testOnlyT1ShouldFireFixedWindows
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testOnlyT1ShouldFireFixedWindows() throws Exception {
tester =
TriggerStateMachineTester.forTrigger(
AfterFirstStateMachine.of(mockTrigger1, mockTrigger2),
FixedWindows.of(Duration.millis(10)));
tester.injectElements(1);
IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11));
when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true);
when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false);
assertTrue(tester.shouldFire(window)); // should fire
tester.fireIfShouldFire(window);
assertTrue(tester.isMarkedFinished(window));
}
示例7: testReshuffleAfterFixedWindowsAndGroupByKey
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterFixedWindowsAndGroupByKey() {
PCollection<KV<String, Iterable<Integer>>> input = pipeline
.apply(Create.of(GBK_TESTABLE_KVS)
.withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
.apply(Window.<KV<String, Integer>>into(
FixedWindows.of(Duration.standardMinutes(10L))))
.apply(GroupByKey.<String, Integer>create());
PCollection<KV<String, Iterable<Integer>>> output = input
.apply(Reshuffle.<String, Iterable<Integer>>of());
PAssert.that(output).satisfies(new AssertThatHasExpectedContents());
assertEquals(
input.getWindowingStrategy(),
output.getWindowingStrategy());
pipeline.run();
}
示例8: testReshuffleAfterFixedWindows
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterFixedWindows() {
PCollection<KV<String, Integer>> input = pipeline
.apply(Create.of(ARBITRARY_KVS)
.withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
.apply(Window.<KV<String, Integer>>into(
FixedWindows.of(Duration.standardMinutes(10L))));
PCollection<KV<String, Integer>> output = input
.apply(Reshuffle.<String, Integer>of());
PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS);
assertEquals(
input.getWindowingStrategy(),
output.getWindowingStrategy());
pipeline.run();
}
示例9: testReshuffleAfterSlidingWindows
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterSlidingWindows() {
PCollection<KV<String, Integer>> input = pipeline
.apply(Create.of(ARBITRARY_KVS)
.withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
.apply(Window.<KV<String, Integer>>into(
FixedWindows.of(Duration.standardMinutes(10L))));
PCollection<KV<String, Integer>> output = input
.apply(Reshuffle.<String, Integer>of());
PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS);
assertEquals(
input.getWindowingStrategy(),
output.getWindowingStrategy());
pipeline.run();
}
示例10: testSampleAnyZero
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testSampleAnyZero() {
PCollection<Integer> input =
pipeline.apply(
Create.timestamped(ImmutableList.of(tv(0), tv(1), tv(2), tv(3), tv(4), tv(5)))
.withCoder(BigEndianIntegerCoder.of()));
PCollection<Integer> output = input
.apply(Window.<Integer>into(FixedWindows.of(Duration.standardSeconds(3))))
.apply(Sample.<Integer>any(0));
PAssert.that(output)
.inWindow(new IntervalWindow(new Instant(0), Duration.standardSeconds(3)))
.satisfies(new VerifyCorrectSample<>(0, EMPTY));
PAssert.that(output)
.inWindow(new IntervalWindow(new Instant(3000), Duration.standardSeconds(3)))
.satisfies(new VerifyCorrectSample<>(0, EMPTY));
pipeline.run();
}
示例11: testActualFiresAndFinishes
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
/**
* Tests that for {@code OrFinally(actual, ...)} when {@code actual}
* fires and finishes, the {@code OrFinally} also fires and finishes.
*/
@Test
public void testActualFiresAndFinishes() throws Exception {
tester = TriggerStateMachineTester.forTrigger(
new OrFinallyStateMachine(
AfterPaneStateMachine.elementCountAtLeast(2),
AfterPaneStateMachine.elementCountAtLeast(100)),
FixedWindows.of(Duration.millis(100)));
IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
// Not yet firing
tester.injectElements(1);
assertFalse(tester.shouldFire(window));
assertFalse(tester.isMarkedFinished(window));
// The actual fires and finishes
tester.injectElements(2);
assertTrue(tester.shouldFire(window));
tester.fireIfShouldFire(window);
assertTrue(tester.isMarkedFinished(window));
}
示例12: testT1FiresFirst
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testT1FiresFirst() throws Exception {
tester = TriggerStateMachineTester.forTrigger(
AfterAllStateMachine.of(
AfterPaneStateMachine.elementCountAtLeast(1),
AfterPaneStateMachine.elementCountAtLeast(2)),
FixedWindows.of(Duration.millis(100)));
IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
tester.injectElements(1);
assertFalse(tester.shouldFire(window));
tester.injectElements(2);
assertTrue(tester.shouldFire(window));
tester.fireIfShouldFire(window);
assertTrue(tester.isMarkedFinished(window));
}
示例13: testEqualWindowFnPropagation
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testEqualWindowFnPropagation() {
PCollection<String> input1 =
p.apply("CreateInput1", Create.of("Input1"))
.apply("Window1", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
PCollection<String> input2 =
p.apply("CreateInput2", Create.of("Input2"))
.apply("Window2", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
PCollection<String> output =
PCollectionList.of(input1).and(input2)
.apply(Flatten.<String>pCollections());
p.run();
Assert.assertTrue(output.getWindowingStrategy().getWindowFn().isCompatible(
FixedWindows.of(Duration.standardMinutes(1))));
}
示例14: testIncompatibleWindowFnPropagationFailure
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testIncompatibleWindowFnPropagationFailure() {
p.enableAbandonedNodeEnforcement(false);
PCollection<String> input1 =
p.apply("CreateInput1", Create.of("Input1"))
.apply("Window1", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
PCollection<String> input2 =
p.apply("CreateInput2", Create.of("Input2"))
.apply("Window2", Window.<String>into(FixedWindows.of(Duration.standardMinutes(2))));
try {
PCollectionList.of(input1).and(input2)
.apply(Flatten.<String>pCollections());
Assert.fail("Exception should have been thrown");
} catch (IllegalStateException e) {
Assert.assertTrue(e.getMessage().startsWith(
"Inputs to Flatten had incompatible window windowFns"));
}
}
示例15: testIdentityWindowFnPropagation
import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testIdentityWindowFnPropagation() {
List<KV<String, Integer>> ungroupedPairs = Arrays.asList();
PCollection<KV<String, Integer>> input =
p.apply(Create.of(ungroupedPairs)
.withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
.apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.standardMinutes(1))));
PCollection<KV<String, Iterable<Integer>>> output =
input.apply(GroupByKey.<String, Integer>create());
p.run();
Assert.assertTrue(output.getWindowingStrategy().getWindowFn().isCompatible(
FixedWindows.of(Duration.standardMinutes(1))));
}