Java FixedWindows类代码示例

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

    Options options = PipelineOptionsFactory.fromArgs(args).withValidation()
        .as(Options.class);
    options.setRunner(FlinkRunner.class);

    Pipeline p = Pipeline.create(options);

    KafkaIO.Read<byte[], String> kafkaIOReader = KafkaIO.read()
        .withBootstrapServers("192.168.99.100:32771")
        .withTopics(Arrays.asList("beam".split(",")))
        .updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest"))
        .withValueCoder(StringUtf8Coder.of());

    p.apply(kafkaIOReader.withoutMetadata())
        .apply(Values.<String>create())
        .apply(Window.<String>into(
          FixedWindows.of(Duration.standardMinutes(options.getWindowSize()))))
        .apply(new CountWords())
        .apply(MapElements.via(new FormatAsTextFn()))
        .apply("WriteCounts", TextIO.Write.to(options.getOutput()));

    p.run();
  }
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
/** Run a batch pipeline to calculate hourly team scores. */
public static void main(String[] args) throws Exception {

  Options options =
      PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline pipeline = Pipeline.create(options);

  pipeline
  .apply("ReadLogs", TextIO.read().from(options.getInput()))
  .apply("SetTimestamps", WithTimestamps.of(new SetTimestampFn()))

  .apply("FixedWindows", Window.<String>into(FixedWindows.of(ONE_HOUR)))

  .apply("TeamScores", new CalculateTeamScores(options.getOutputPrefix()));

  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

    Options options =
        PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
    Pipeline pipeline = Pipeline.create(options);

    pipeline
    .apply(KafkaIO.<String, String>read()
        .withBootstrapServers(options.getKafkaBootstrapServer())
        .withTopic(options.getTopic())
        .withKeyDeserializer(StringDeserializer.class)
        .withValueDeserializer(StringDeserializer.class)
        .withTimestampFn(new SetTimestampFn()))
    .apply("Values", ParDo.of(new ValuesFn()))

    .apply("FixedWindows", Window.<String>into(FixedWindows.of(FIVE_MINUTES))
        .triggering(AfterWatermark.pastEndOfWindow()
            .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
                .plusDelayOf(TWO_MINUTES))
            .withLateFirings(AfterPane.elementCountAtLeast(1)))
        .withAllowedLateness(TEN_MINUTES)
        .accumulatingFiredPanes())

    .apply("TeamScore", new CalculateTeamScores(options.getOutputPrefix()));

    pipeline.run();
  }
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Override
public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> infos) {
  return infos.apply("LeaderboardTeamFixedWindows",
      Window.<GameActionInfo>into(FixedWindows.of(teamWindowDuration))
          // We will get early (speculative) results as well as cumulative
          // processing of late data.
          .triggering(AfterWatermark.pastEndOfWindow()
              .withEarlyFirings(AfterProcessingTime.pastFirstElementInPane()
                  .plusDelayOf(FIVE_MINUTES))
              .withLateFirings(AfterProcessingTime.pastFirstElementInPane()
                  .plusDelayOf(TEN_MINUTES)))
          .withAllowedLateness(allowedLateness)
          .accumulatingFiredPanes())
      // Extract and sum teamname/score pairs from the event data.
      .apply("ExtractTeamScore", new ExtractAndSumScore("team"));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testTotalFlow () {
  PCollection<KV<String, Integer>> flow = pipeline
      .apply(Create.timestamped(TIME_STAMPED_INPUT))
      .apply(ParDo.of(new ExtractFlowInfo()));

  PCollection<TableRow> totalFlow = flow
      .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.standardMinutes(1))))
      .apply(new TotalFlow("default"));

  PCollection<String> results =  totalFlow.apply(ParDo.of(new FormatResults()));

  PAssert.that(results)
      .containsInAnyOrder(canonicalFormat(OUT_ROW_1), canonicalFormat(OUT_ROW_2));
  pipeline.run().waitUntilFinish();

}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testOnlyT1ShouldFireFixedWindows() throws Exception {
  tester =
      TriggerStateMachineTester.forTrigger(
          AfterFirstStateMachine.of(mockTrigger1, mockTrigger2),
          FixedWindows.of(Duration.millis(10)));
  tester.injectElements(1);
  IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11));

  when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true);
  when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false);

  assertTrue(tester.shouldFire(window)); // should fire

  tester.fireIfShouldFire(window);
  assertTrue(tester.isMarkedFinished(window));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterFixedWindowsAndGroupByKey() {

  PCollection<KV<String, Iterable<Integer>>> input = pipeline
      .apply(Create.of(GBK_TESTABLE_KVS)
          .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
      .apply(Window.<KV<String, Integer>>into(
          FixedWindows.of(Duration.standardMinutes(10L))))
      .apply(GroupByKey.<String, Integer>create());

  PCollection<KV<String, Iterable<Integer>>> output = input
      .apply(Reshuffle.<String, Iterable<Integer>>of());

  PAssert.that(output).satisfies(new AssertThatHasExpectedContents());

  assertEquals(
      input.getWindowingStrategy(),
      output.getWindowingStrategy());

  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterFixedWindows() {

  PCollection<KV<String, Integer>> input = pipeline
      .apply(Create.of(ARBITRARY_KVS)
          .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
      .apply(Window.<KV<String, Integer>>into(
          FixedWindows.of(Duration.standardMinutes(10L))));

  PCollection<KV<String, Integer>> output = input
      .apply(Reshuffle.<String, Integer>of());

  PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS);

  assertEquals(
      input.getWindowingStrategy(),
      output.getWindowingStrategy());

  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testReshuffleAfterSlidingWindows() {

  PCollection<KV<String, Integer>> input = pipeline
      .apply(Create.of(ARBITRARY_KVS)
          .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))
      .apply(Window.<KV<String, Integer>>into(
          FixedWindows.of(Duration.standardMinutes(10L))));

  PCollection<KV<String, Integer>> output = input
      .apply(Reshuffle.<String, Integer>of());

  PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS);

  assertEquals(
      input.getWindowingStrategy(),
      output.getWindowingStrategy());

  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testSampleAnyZero() {
  PCollection<Integer> input =
      pipeline.apply(
          Create.timestamped(ImmutableList.of(tv(0), tv(1), tv(2), tv(3), tv(4), tv(5)))
              .withCoder(BigEndianIntegerCoder.of()));
  PCollection<Integer> output = input
      .apply(Window.<Integer>into(FixedWindows.of(Duration.standardSeconds(3))))
      .apply(Sample.<Integer>any(0));

  PAssert.that(output)
      .inWindow(new IntervalWindow(new Instant(0), Duration.standardSeconds(3)))
      .satisfies(new VerifyCorrectSample<>(0, EMPTY));
  PAssert.that(output)
      .inWindow(new IntervalWindow(new Instant(3000), Duration.standardSeconds(3)))
      .satisfies(new VerifyCorrectSample<>(0, EMPTY));
  pipeline.run();
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
/**
 * Tests that for {@code OrFinally(actual, ...)} when {@code actual}
 * fires and finishes, the {@code OrFinally} also fires and finishes.
 */
@Test
public void testActualFiresAndFinishes() throws Exception {
  tester = TriggerStateMachineTester.forTrigger(
      new OrFinallyStateMachine(
          AfterPaneStateMachine.elementCountAtLeast(2),
          AfterPaneStateMachine.elementCountAtLeast(100)),
      FixedWindows.of(Duration.millis(100)));

  IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));

  // Not yet firing
  tester.injectElements(1);
  assertFalse(tester.shouldFire(window));
  assertFalse(tester.isMarkedFinished(window));

  // The actual fires and finishes
  tester.injectElements(2);
  assertTrue(tester.shouldFire(window));
  tester.fireIfShouldFire(window);
  assertTrue(tester.isMarkedFinished(window));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testT1FiresFirst() throws Exception {
  tester = TriggerStateMachineTester.forTrigger(
      AfterAllStateMachine.of(
          AfterPaneStateMachine.elementCountAtLeast(1),
          AfterPaneStateMachine.elementCountAtLeast(2)),
      FixedWindows.of(Duration.millis(100)));

  IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));

  tester.injectElements(1);
  assertFalse(tester.shouldFire(window));

  tester.injectElements(2);
  assertTrue(tester.shouldFire(window));
  tester.fireIfShouldFire(window);
  assertTrue(tester.isMarkedFinished(window));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testEqualWindowFnPropagation() {
  PCollection<String> input1 =
      p.apply("CreateInput1", Create.of("Input1"))
      .apply("Window1", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
  PCollection<String> input2 =
      p.apply("CreateInput2", Create.of("Input2"))
      .apply("Window2", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));

  PCollection<String> output =
      PCollectionList.of(input1).and(input2)
      .apply(Flatten.<String>pCollections());

  p.run();

  Assert.assertTrue(output.getWindowingStrategy().getWindowFn().isCompatible(
      FixedWindows.of(Duration.standardMinutes(1))));
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
public void testIncompatibleWindowFnPropagationFailure() {
  p.enableAbandonedNodeEnforcement(false);

  PCollection<String> input1 =
      p.apply("CreateInput1", Create.of("Input1"))
      .apply("Window1", Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
  PCollection<String> input2 =
      p.apply("CreateInput2", Create.of("Input2"))
      .apply("Window2", Window.<String>into(FixedWindows.of(Duration.standardMinutes(2))));

  try {
    PCollectionList.of(input1).and(input2)
        .apply(Flatten.<String>pCollections());
    Assert.fail("Exception should have been thrown");
  } catch (IllegalStateException e) {
    Assert.assertTrue(e.getMessage().startsWith(
        "Inputs to Flatten had incompatible window windowFns"));
  }
}
 

import org.apache.beam.sdk.transforms.windowing.FixedWindows; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testIdentityWindowFnPropagation() {

  List<KV<String, Integer>> ungroupedPairs = Arrays.asList();

  PCollection<KV<String, Integer>> input =
      p.apply(Create.of(ungroupedPairs)
          .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
      .apply(Window.<KV<String, Integer>>into(FixedWindows.of(Duration.standardMinutes(1))));

  PCollection<KV<String, Iterable<Integer>>> output =
      input.apply(GroupByKey.<String, Integer>create());

  p.run();

  Assert.assertTrue(output.getWindowingStrategy().getWindowFn().isCompatible(
      FixedWindows.of(Duration.standardMinutes(1))));
}