Java Windows类代码示例

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEventStream", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, PageViewCount>> pageViewEventPerMemberStream =
      graph.getOutputStream("pageViewEventPerMemberStream",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewCount.class)));

  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> foldLeftFn = (m, c) -> c + 1;
  pageViewEvents
      .window(Windows.keyedTumblingWindow(m -> m.memberId, Duration.ofSeconds(10), initialValue, foldLeftFn, null, null)
          .setEarlyTrigger(Triggers.repeat(Triggers.count(5)))
          .setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), new PageViewCount(windowPane)))
      .sendTo(pageViewEventPerMemberStream);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> counter = (m, c) -> c == null ? 1 : c + 1;
  MessageStream<PageViewEvent> inputStream = graph.getInputStream("inputStream", new JsonSerdeV2<PageViewEvent>());
  OutputStream<Integer> outputStream = graph.getOutputStream("outputStream", new IntegerSerde());

  // create a tumbling window that outputs the number of message collected every 10 minutes.
  // also emit early results if either the number of messages collected reaches 30000, or if no new messages arrive
  // for 1 minute.
  inputStream
      .window(Windows.tumblingWindow(Duration.ofMinutes(10), initialValue, counter, new IntegerSerde())
          .setLateTrigger(Triggers.any(Triggers.count(30000),
              Triggers.timeSinceLastMessage(Duration.ofMinutes(1)))), "window")
      .map(WindowPane::getMessage)
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Test
public void testWindowWithRelaxedTypes() throws Exception {
  StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
  OperatorSpec mockOpSpec = mock(OperatorSpec.class);
  MessageStream<TestInputMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);

  Function<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
  FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
  Supplier<Integer> initialValue = () -> 0;

  // should compile since TestMessageEnvelope (input for functions) is base class of TestInputMessageEnvelope (M)
  Window<TestInputMessageEnvelope, String, Integer> window =
      Windows.keyedTumblingWindow(keyExtractor, Duration.ofHours(1), initialValue, aggregator,
          null, mock(Serde.class));
  MessageStream<WindowPane<String, Integer>> windowedStream = inputStream.window(window, "w1");

  ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
  verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
  OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();

  assertTrue(registeredOpSpec instanceof WindowOperatorSpec);
  assertEquals(OpCode.WINDOW, registeredOpSpec.getOpCode());
  assertEquals(window, ((WindowOperatorSpec) registeredOpSpec).getWindow());
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, MyStreamOutput>> pageViewEventPerMember =
      graph.getOutputStream("pageViewEventPerMember",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(MyStreamOutput.class)));

  pageViewEvents
      .partitionBy(pve -> pve.memberId, pve -> pve,
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewEvent.class)), "partitionBy")
      .window(Windows.keyedTumblingWindow(KV::getKey, Duration.ofMinutes(5), () -> 0, (m, c) -> c + 1, null, null),
          "window")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), new MyStreamOutput(windowPane)))
      .sendTo(pageViewEventPerMember);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<IntegerEnvelope> inStream =
      graph.getInputStream("integers", KVSerde.of(new IntegerSerde(), new IntegerSerde()))
          .map(kv -> new IntegerEnvelope(kv.getKey()));
  Function<IntegerEnvelope, Integer> keyFn = m -> (Integer) m.getKey();
  inStream
      .map(m -> m)
      .window(Windows.keyedTumblingWindow(keyFn, duration, new IntegerSerde(), new IntegerEnvelopeSerde())
          .setEarlyTrigger(earlyTrigger)
          .setAccumulationMode(mode), "w1")
      .sink((message, messageCollector, taskCoordinator) -> {
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<PageView> pageViews = graph.getInputStream(INPUT_TOPIC, new JsonSerdeV2<>(PageView.class));
  OutputStream<KV<String, Integer>> outputStream =
      graph.getOutputStream(OUTPUT_TOPIC, new KVSerde<>(new StringSerde(), new IntegerSerde()));

  pageViews
      .filter(m -> !FILTER_KEY.equals(m.getUserId()))
      .window(Windows.keyedSessionWindow(PageView::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(PageView.class)), "sessionWindow")
      .map(m -> KV.of(m.getKey().getKey(), m.getMessage().size()))
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<PageView> pageViews =
      graph.getInputStream(INPUT_TOPIC, new JsonSerdeV2<>(PageView.class));
  OutputStream<KV<String, Integer>> outputStream =
      graph.getOutputStream(OUTPUT_TOPIC, new KVSerde<>(new StringSerde(), new IntegerSerde()));

  pageViews
      .filter(m -> !FILTER_KEY.equals(m.getUserId()))
      .window(Windows.keyedTumblingWindow(PageView::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(PageView.class)), "tumblingWindow")
      .map(m -> KV.of(m.getKey().getKey(), m.getMessage().size()))
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<PageView> pageViews = graph.getInputStream(PAGE_VIEWS, new JsonSerdeV2<>(PageView.class));
  MessageStream<AdClick> adClicks = graph.getInputStream(AD_CLICKS, new JsonSerdeV2<>(AdClick.class));
  OutputStream<KV<String, String>> outputStream =
      graph.getOutputStream(OUTPUT_TOPIC, new KVSerde<>(new StringSerde(), new StringSerde()));

  MessageStream<PageView> pageViewsRepartitionedByViewId = pageViews
      .partitionBy(PageView::getViewId, pv -> pv,
          new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(PageView.class)), "pageViewsByViewId")
      .map(KV::getValue);

  MessageStream<AdClick> adClicksRepartitionedByViewId = adClicks
      .partitionBy(AdClick::getViewId, ac -> ac,
          new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(AdClick.class)), "adClicksByViewId")
      .map(KV::getValue);

  MessageStream<UserPageAdClick> userPageAdClicks = pageViewsRepartitionedByViewId
      .join(adClicksRepartitionedByViewId, new UserPageViewAdClicksJoiner(),
          new StringSerde(), new JsonSerdeV2<>(PageView.class), new JsonSerdeV2<>(AdClick.class),
          Duration.ofMinutes(1), "pageViewAdClickJoin");

  userPageAdClicks
      .partitionBy(UserPageAdClick::getUserId, upac -> upac,
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userPageAdClicksByUserId")
      .map(KV::getValue)
      .window(Windows.keyedSessionWindow(UserPageAdClick::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userAdClickWindow")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), String.valueOf(windowPane.getMessage().size())))
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  // Messages come from WikipediaConsumer so we know that they don't have a key and don't need to be deserialized.
  graph.setDefaultSerde(new NoOpSerde<>());

  // Inputs
  // Messages come from WikipediaConsumer so we know the type is WikipediaFeedEvent
  MessageStream<WikipediaFeedEvent> wikipediaEvents = graph.getInputStream(WIKIPEDIA_STREAM_ID);
  MessageStream<WikipediaFeedEvent> wiktionaryEvents = graph.getInputStream(WIKTIONARY_STREAM_ID);
  MessageStream<WikipediaFeedEvent> wikiNewsEvents = graph.getInputStream(WIKINEWS_STREAM_ID);

  // Output (also un-keyed)
  OutputStream<WikipediaStatsOutput> wikipediaStats =
      graph.getOutputStream(STATS_STREAM_ID, new JsonSerdeV2<>(WikipediaStatsOutput.class));

  // Merge inputs
  MessageStream<WikipediaFeedEvent> allWikipediaEvents =
      MessageStream.mergeAll(ImmutableList.of(wikipediaEvents, wiktionaryEvents, wikiNewsEvents));

  // Parse, update stats, prepare output, and send
  allWikipediaEvents
      .map(WikipediaParser::parseEvent)
      .window(Windows.tumblingWindow(Duration.ofSeconds(10), WikipediaStats::new,
              new WikipediaStatsAggregator(), WikipediaStats.serde()), "statsWindow")
      .map(this::formatOutput)
      .sendTo(wikipediaStats);
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
@Test
public void test2() throws Exception {
  Map<String, String> configMap = new HashMap<>();
  configMap.put(JobConfig.JOB_NAME(), "test-app");
  configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system");
  Config config = new MapConfig(configMap);

  StreamSpec input = new StreamSpec("PageView", "hdfs:/user/dummy/PageViewEvent", "hdfs");
  StreamSpec output = new StreamSpec("PageViewCount", "PageViewCount", "kafka");

  ApplicationRunner runner = mock(ApplicationRunner.class);
  when(runner.getStreamSpec("PageView")).thenReturn(input);
  when(runner.getStreamSpec("PageViewCount")).thenReturn(output);

  // intermediate streams used in tests
  when(runner.getStreamSpec("test-app-1-partition_by-keyed-by-country"))
      .thenReturn(new StreamSpec("test-app-1-partition_by-keyed-by-country", "test-app-1-partition_by-keyed-by-country", "kafka"));

  // set up external partition count
  Map<String, Integer> system1Map = new HashMap<>();
  system1Map.put("hdfs:/user/dummy/PageViewEvent", 512);
  Map<String, Integer> system2Map = new HashMap<>();
  system2Map.put("PageViewCount", 16);

  Map<String, SystemAdmin> systemAdmins = new HashMap<>();
  SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
  SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
  systemAdmins.put("hdfs", systemAdmin1);
  systemAdmins.put("kafka", systemAdmin2);
  StreamManager streamManager = new StreamManager(systemAdmins);

  StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
  MessageStream<KV<String, PageViewEvent>> inputStream = streamGraph.getInputStream("PageView");
  inputStream
      .partitionBy(kv -> kv.getValue().getCountry(), kv -> kv.getValue(), "keyed-by-country")
      .window(Windows.keyedTumblingWindow(kv -> kv.getValue().getCountry(),
          Duration.ofSeconds(10L),
          () -> 0L,
          (m, c) -> c + 1L,
          new StringSerde(),
          new LongSerde()), "count-by-country")
      .map(pane -> new KV<>(pane.getKey().getKey(), pane.getMessage()))
      .sendTo(streamGraph.getOutputStream("PageViewCount"));

  ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
  ExecutionPlan plan = planner.plan(streamGraph);
  String json = plan.getPlanAsJson();
  System.out.println(json);

  // deserialize
  ObjectMapper mapper = new ObjectMapper();
  JobGraphJsonGenerator.JobGraphJson nodes = mapper.readValue(json, JobGraphJsonGenerator.JobGraphJson.class);
  JobGraphJsonGenerator.OperatorGraphJson operatorGraphJson = nodes.jobs.get(0).operatorGraph;
  assertEquals(2, operatorGraphJson.inputStreams.size());
  assertEquals(4, operatorGraphJson.operators.size());
  assertEquals(1, nodes.sourceStreams.size());
  assertEquals(1, nodes.sinkStreams.size());
  assertEquals(1, nodes.intermediateStreams.size());

  // verify partitionBy op output to the intermdiate stream of the same id
  assertEquals(operatorGraphJson.operators.get("test-app-1-partition_by-keyed-by-country").get("outputStreamId"),
      "test-app-1-partition_by-keyed-by-country");
  assertEquals(operatorGraphJson.operators.get("test-app-1-send_to-5").get("outputStreamId"),
      "PageViewCount");
}
 

import org.apache.samza.operators.windows.Windows; //导入依赖的package包/类
private StreamGraphImpl createStreamGraphWithJoinAndWindow() {

    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
    MessageStream<KV<Object, Object>> messageStream1 =
        streamGraph.<KV<Object, Object>>getInputStream("input1")
            .map(m -> m);
    MessageStream<KV<Object, Object>> messageStream2 =
        streamGraph.<KV<Object, Object>>getInputStream("input2")
            .partitionBy(m -> m.key, m -> m.value, "p1")
            .filter(m -> true);
    MessageStream<KV<Object, Object>> messageStream3 =
        streamGraph.<KV<Object, Object>>getInputStream("input3")
            .filter(m -> true)
            .partitionBy(m -> m.key, m -> m.value, "p2")
            .map(m -> m);
    OutputStream<KV<Object, Object>> output1 = streamGraph.getOutputStream("output1");
    OutputStream<KV<Object, Object>> output2 = streamGraph.getOutputStream("output2");

    messageStream1.map(m -> m)
        .filter(m->true)
        .window(Windows.<KV<Object, Object>, Object>keyedTumblingWindow(m -> m, Duration.ofMillis(8),
            mock(Serde.class), mock(Serde.class)), "w1");

    messageStream2.map(m -> m)
        .filter(m->true)
        .window(Windows.<KV<Object, Object>, Object>keyedTumblingWindow(m -> m, Duration.ofMillis(16),
            mock(Serde.class), mock(Serde.class)), "w2");

    messageStream1
        .join(messageStream2, mock(JoinFunction.class),
            mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(1600), "j1")
        .sendTo(output1);
    messageStream3
        .join(messageStream2, mock(JoinFunction.class),
            mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(100), "j2")
        .sendTo(output2);
    messageStream3
        .join(messageStream2, mock(JoinFunction.class),
            mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(252), "j3")
        .sendTo(output2);

    return streamGraph;
  }