Java MessageStream.join方法代码示例

import org.apache.samza.operators.MessageStream; //导入方法依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<PageView> pageViews = graph.getInputStream(PAGE_VIEWS, new JsonSerdeV2<>(PageView.class));
  MessageStream<AdClick> adClicks = graph.getInputStream(AD_CLICKS, new JsonSerdeV2<>(AdClick.class));
  OutputStream<KV<String, String>> outputStream =
      graph.getOutputStream(OUTPUT_TOPIC, new KVSerde<>(new StringSerde(), new StringSerde()));

  MessageStream<PageView> pageViewsRepartitionedByViewId = pageViews
      .partitionBy(PageView::getViewId, pv -> pv,
          new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(PageView.class)), "pageViewsByViewId")
      .map(KV::getValue);

  MessageStream<AdClick> adClicksRepartitionedByViewId = adClicks
      .partitionBy(AdClick::getViewId, ac -> ac,
          new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(AdClick.class)), "adClicksByViewId")
      .map(KV::getValue);

  MessageStream<UserPageAdClick> userPageAdClicks = pageViewsRepartitionedByViewId
      .join(adClicksRepartitionedByViewId, new UserPageViewAdClicksJoiner(),
          new StringSerde(), new JsonSerdeV2<>(PageView.class), new JsonSerdeV2<>(AdClick.class),
          Duration.ofMinutes(1), "pageViewAdClickJoin");

  userPageAdClicks
      .partitionBy(UserPageAdClick::getUserId, upac -> upac,
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userPageAdClicksByUserId")
      .map(KV::getValue)
      .window(Windows.keyedSessionWindow(UserPageAdClick::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userAdClickWindow")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), String.valueOf(windowPane.getMessage().size())))
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.MessageStream; //导入方法依赖的package包/类
@Test
public void testJoinChain() {
  ApplicationRunner mockRunner = mock(ApplicationRunner.class);
  when(mockRunner.getStreamSpec(eq("input1"))).thenReturn(new StreamSpec("input1", "input-stream1", "input-system"));
  when(mockRunner.getStreamSpec(eq("input2"))).thenReturn(new StreamSpec("input2", "input-stream2", "input-system"));
  Config mockConfig = mock(Config.class);
  when(mockConfig.get(JobConfig.JOB_NAME())).thenReturn("jobName");
  when(mockConfig.get(eq(JobConfig.JOB_ID()), anyString())).thenReturn("jobId");
  StreamGraphImpl streamGraph = new StreamGraphImpl(mockRunner, mockConfig);

  JoinFunction mockJoinFunction = mock(JoinFunction.class);
  MessageStream<Object> inputStream1 = streamGraph.getInputStream("input1", new NoOpSerde<>());
  MessageStream<Object> inputStream2 = streamGraph.getInputStream("input2", new NoOpSerde<>());
  inputStream1.join(inputStream2, mockJoinFunction,
      mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j1");

  TaskContextImpl mockTaskContext = mock(TaskContextImpl.class);
  when(mockTaskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
  KeyValueStore mockLeftStore = mock(KeyValueStore.class);
  when(mockTaskContext.getStore(eq("jobName-jobId-join-j1-L"))).thenReturn(mockLeftStore);
  KeyValueStore mockRightStore = mock(KeyValueStore.class);
  when(mockTaskContext.getStore(eq("jobName-jobId-join-j1-R"))).thenReturn(mockRightStore);
  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(streamGraph, mockConfig, mockTaskContext, mock(Clock.class));

  // verify that join function is initialized once.
  verify(mockJoinFunction, times(1)).init(any(Config.class), any(TaskContextImpl.class));

  InputOperatorImpl inputOpImpl1 = opImplGraph.getInputOperator(new SystemStream("input-system", "input-stream1"));
  InputOperatorImpl inputOpImpl2 = opImplGraph.getInputOperator(new SystemStream("input-system", "input-stream2"));
  PartialJoinOperatorImpl leftPartialJoinOpImpl =
      (PartialJoinOperatorImpl) inputOpImpl1.registeredOperators.iterator().next();
  PartialJoinOperatorImpl rightPartialJoinOpImpl =
      (PartialJoinOperatorImpl) inputOpImpl2.registeredOperators.iterator().next();

  assertEquals(leftPartialJoinOpImpl.getOperatorSpec(), rightPartialJoinOpImpl.getOperatorSpec());
  assertNotSame(leftPartialJoinOpImpl, rightPartialJoinOpImpl);

  Object joinKey = new Object();
  // verify that left partial join operator calls getFirstKey
  Object mockLeftMessage = mock(Object.class);
  long currentTimeMillis = System.currentTimeMillis();
  when(mockLeftStore.get(eq(joinKey))).thenReturn(new TimestampedValue<>(mockLeftMessage, currentTimeMillis));
  when(mockJoinFunction.getFirstKey(eq(mockLeftMessage))).thenReturn(joinKey);
  inputOpImpl1.onMessage(KV.of("", mockLeftMessage), mock(MessageCollector.class), mock(TaskCoordinator.class));
  verify(mockJoinFunction, times(1)).getFirstKey(mockLeftMessage);

  // verify that right partial join operator calls getSecondKey
  Object mockRightMessage = mock(Object.class);
  when(mockRightStore.get(eq(joinKey))).thenReturn(new TimestampedValue<>(mockRightMessage, currentTimeMillis));
  when(mockJoinFunction.getSecondKey(eq(mockRightMessage))).thenReturn(joinKey);
  inputOpImpl2.onMessage(KV.of("", mockRightMessage), mock(MessageCollector.class), mock(TaskCoordinator.class));
  verify(mockJoinFunction, times(1)).getSecondKey(mockRightMessage);

  // verify that the join function apply is called with the correct messages on match
  verify(mockJoinFunction, times(1)).apply(mockLeftMessage, mockRightMessage);
}