Java MessageStream类代码示例

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<OrderRecord> orders =
      graph.getInputStream("orders", new JsonSerdeV2<>(OrderRecord.class));
  MessageStream<ShipmentRecord> shipments =
      graph.getInputStream("shipments", new JsonSerdeV2<>(ShipmentRecord.class));
  OutputStream<KV<String, FulfilledOrderRecord>> fulfilledOrders =
      graph.getOutputStream("fulfilledOrders",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(FulfilledOrderRecord.class)));

  orders
      .join(shipments, new MyJoinFunction(),
          new StringSerde(), new JsonSerdeV2<>(OrderRecord.class), new JsonSerdeV2<>(ShipmentRecord.class),
          Duration.ofMinutes(1), "join")
      .map(fulFilledOrder -> KV.of(fulFilledOrder.orderId, fulFilledOrder))
      .sendTo(fulfilledOrders);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEventStream", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, PageViewCount>> pageViewEventPerMemberStream =
      graph.getOutputStream("pageViewEventPerMemberStream",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewCount.class)));

  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> foldLeftFn = (m, c) -> c + 1;
  pageViewEvents
      .window(Windows.keyedTumblingWindow(m -> m.memberId, Duration.ofSeconds(10), initialValue, foldLeftFn, null, null)
          .setEarlyTrigger(Triggers.repeat(Triggers.count(5)))
          .setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), new PageViewCount(windowPane)))
      .sendTo(pageViewEventPerMemberStream);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> counter = (m, c) -> c == null ? 1 : c + 1;
  MessageStream<PageViewEvent> inputStream = graph.getInputStream("inputStream", new JsonSerdeV2<PageViewEvent>());
  OutputStream<Integer> outputStream = graph.getOutputStream("outputStream", new IntegerSerde());

  // create a tumbling window that outputs the number of message collected every 10 minutes.
  // also emit early results if either the number of messages collected reaches 30000, or if no new messages arrive
  // for 1 minute.
  inputStream
      .window(Windows.tumblingWindow(Duration.ofMinutes(10), initialValue, counter, new IntegerSerde())
          .setLateTrigger(Triggers.any(Triggers.count(30000),
              Triggers.timeSinceLastMessage(Duration.ofMinutes(1)))), "window")
      .map(WindowPane::getMessage)
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
private StreamGraphImpl createSimpleGraph() {
  /**
   * a simple graph of partitionBy and map
   *
   * input1 -> partitionBy -> map -> output1
   *
   */
  StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
  MessageStream<KV<Object, Object>> input1 = streamGraph.getInputStream("input1");
  OutputStream<KV<Object, Object>> output1 = streamGraph.getOutputStream("output1");
  input1
      .partitionBy(m -> m.key, m -> m.value, "p1")
      .map(kv -> kv)
      .sendTo(output1);
  return streamGraph;
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Test
public void testMaxPartitionLimit() throws Exception {
  int partitionLimit = ExecutionPlanner.MAX_INFERRED_PARTITIONS;

  ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
  StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);

  MessageStream<KV<Object, Object>> input1 = streamGraph.getInputStream("input4");
  OutputStream<KV<Object, Object>> output1 = streamGraph.getOutputStream("output1");
  input1.partitionBy(m -> m.key, m -> m.value, "p1").map(kv -> kv).sendTo(output1);
  JobGraph jobGraph = (JobGraph) planner.plan(streamGraph);

  // the partitions should be the same as input1
  jobGraph.getIntermediateStreams().forEach(edge -> {
      assertEquals(partitionLimit, edge.getPartitionCount()); // max of input1 and output1
    });
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Test
public void testBroadcastChain() {
  ApplicationRunner mockRunner = mock(ApplicationRunner.class);
  when(mockRunner.getStreamSpec(eq("input"))).thenReturn(new StreamSpec("input", "input-stream", "input-system"));
  StreamGraphImpl streamGraph = new StreamGraphImpl(mockRunner, mock(Config.class));

  MessageStream<Object> inputStream = streamGraph.getInputStream("input");
  inputStream.filter(mock(FilterFunction.class));
  inputStream.map(mock(MapFunction.class));

  TaskContextImpl mockTaskContext = mock(TaskContextImpl.class);
  when(mockTaskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(streamGraph, mock(Config.class), mockTaskContext, mock(Clock.class));

  InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream("input-system", "input-stream"));
  assertEquals(2, inputOpImpl.registeredOperators.size());
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER));
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP));
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Test
public void testMergeChain() {
  ApplicationRunner mockRunner = mock(ApplicationRunner.class);
  when(mockRunner.getStreamSpec(eq("input"))).thenReturn(new StreamSpec("input", "input-stream", "input-system"));
  StreamGraphImpl streamGraph = new StreamGraphImpl(mockRunner, mock(Config.class));

  MessageStream<Object> inputStream = streamGraph.getInputStream("input");
  MessageStream<Object> stream1 = inputStream.filter(mock(FilterFunction.class));
  MessageStream<Object> stream2 = inputStream.map(mock(MapFunction.class));
  MessageStream<Object> mergedStream = stream1.merge(Collections.singleton(stream2));
  MapFunction mockMapFunction = mock(MapFunction.class);
  mergedStream.map(mockMapFunction);

  TaskContextImpl mockTaskContext = mock(TaskContextImpl.class);
  when(mockTaskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(streamGraph, mock(Config.class), mockTaskContext, mock(Clock.class));

  // verify that the DAG after merge is only traversed & initialized once
  verify(mockMapFunction, times(1)).init(any(Config.class), any(TaskContextImpl.class));
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex,
    MessageStream<SamzaSqlRelMessage> inputStream) {
  return inputStream.flatMap(message -> {
    Object field = message.getRelFieldValues().get(flattenIndex);

    if (field != null && field instanceof List) {
      List<SamzaSqlRelMessage> outMessages = new ArrayList<>();
      for (Object fieldValue : (List) field) {
        List<Object> newValues = new ArrayList<>(message.getFieldValues());
        newValues.set(flattenIndex, Collections.singletonList(fieldValue));
        outMessages.add(new SamzaSqlRelMessage(message.getKey(), message.getFieldNames(), newValues));
      }
      return outMessages;
    } else {
      return Collections.singletonList(message);
    }
  });
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
public void translate(final LogicalFilter filter, final TranslatorContext context) {
  MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(filter.getInput().getId());
  Expression expr =
      context.getExpressionCompiler().compile(filter.getInputs(), Collections.singletonList(filter.getCondition()));

  MessageStream<SamzaSqlRelMessage> outputStream = inputStream.filter(message -> {
    Object[] result = new Object[1];
    expr.execute(context.getExecutionContext(), context.getDataContext(), message.getRelFieldValues().toArray(), result);
    if (result.length > 0 && result[0] instanceof Boolean) {
      boolean retVal = (Boolean) result[0];
      log.debug(
          String.format("return value for input %s is %s", Arrays.asList(message.getFieldValues()).toString(), retVal));
      return retVal;
    } else {
      log.error("return value is not boolean");
      return false;
    }
  });

  context.registerMessageStream(filter.getId(), outputStream);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  MessageStream<String> inputStream = graph.getInputStream(inputTopic, new NoOpSerde<String>());
  OutputStream<String> outputStream = graph.getOutputStream(outputTopic, new StringSerde());
  inputStream
      .map(msg -> {
          TestKafkaEvent incomingMessage = TestKafkaEvent.fromString((String) msg);
          if (streamApplicationCallback != null) {
            streamApplicationCallback.onMessageReceived(incomingMessage);
          }
          if (processedMessagesLatch != null) {
            processedMessagesLatch.countDown();
          }
          if (kafkaEventsConsumedLatch != null) {
            kafkaEventsConsumedLatch.countDown();
          }
          return incomingMessage.toString();
        })
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {

  // Input
  MessageStream<KV<String, byte[]>> eventhubInput = graph.getInputStream(INPUT_STREAM_ID);

  // Output
  OutputStream<KV<String, byte[]>> eventhubOutput =
      graph.getOutputStream(OUTPUT_STREAM_ID, KVSerde.of(new StringSerde(), new ByteSerde()));

  // Send
  eventhubInput
      .filter((message) -> message.getKey() != null)
      .map((message) -> {
        System.out.println("Sending: ");
        System.out.println("Received Key: " + message.getKey());
        System.out.println("Received Message: " + new String(message.getValue()));
        return message;
      })
      .sendTo(eventhubOutput);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
/**
 * Get the user source code location that created the operator.
 * @return  source code location for the operator
 */
public final String getSourceLocation() {
  // The stack trace for most operators looks like:
  // [0] Thread.getStackTrace()
  // [1] OperatorSpec.init<>()
  // [2] SomeOperatorSpec.<init>()
  // [3] OperatorSpecs.createSomeOperatorSpec()
  // [4] MessageStreamImpl.someOperator()
  // [5] User/MessageStreamImpl code that calls [4]
  // We are interested in the first call below this that originates from user code
  StackTraceElement element = this.creationStackTrace[5];

  /**
   * Sometimes [5] above is a call from MessageStream/MessageStreamImpl itself (e.g. for
   * {@link org.apache.samza.operators.MessageStream#mergeAll(Collection)} or
   * {@link MessageStreamImpl#partitionBy(Function, Function)}).
   * If that's the case, find the first call from a class other than these.
   */
  for (int i = 5; i < creationStackTrace.length; i++) {
    if (!creationStackTrace[i].getClassName().equals(MessageStreamImpl.class.getName())
        && !creationStackTrace[i].getClassName().equals(MessageStream.class.getName())) {
      element = creationStackTrace[i];
      break;
    }
  }
  return String.format("%s:%s", element.getFileName(), element.getLineNumber());
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, MyStreamOutput>> pageViewEventPerMember =
      graph.getOutputStream("pageViewEventPerMember",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(MyStreamOutput.class)));

  pageViewEvents
      .partitionBy(pve -> pve.memberId, pve -> pve,
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewEvent.class)), "partitionBy")
      .window(Windows.keyedTumblingWindow(KV::getKey, Duration.ofMinutes(5), () -> 0, (m, c) -> c + 1, null, null),
          "window")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), new MyStreamOutput(windowPane)))
      .sendTo(pageViewEventPerMember);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEventStream", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, StatsOutput>> pageViewEventPerMember =
      graph.getOutputStream("pageViewEventPerMember",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(StatsOutput.class)));

  pageViewEvents
      .partitionBy(pve -> pve.memberId, pve -> pve,
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewEvent.class)), "partitionBy")
      .map(KV::getValue)
      .flatMap(new MyStatsCounter())
      .map(stats -> KV.of(stats.memberId, stats))
      .sendTo(pageViewEventPerMember);
}
 

import org.apache.samza.operators.MessageStream; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  graph.setDefaultSerde(new StringSerde());

  MessageStream<String> inputStream1 = graph.getInputStream("inputStream1");
  MessageStream<String> inputStream2 = graph.getInputStream("inputStream2");
  MessageStream<String> inputStream3 = graph.getInputStream("inputStream3");
  OutputStream<KV<Integer, String>> outputStream =
      graph.getOutputStream("outputStream", KVSerde.of(new IntegerSerde(), new StringSerde()));

  MessageStream
      .mergeAll(ImmutableList.of(inputStream1, inputStream2, inputStream3))
      .map(m -> KV.of(m.hashCode(), m))
      .sendTo(outputStream);
}