Java OutgoingMessageEnvelope类代码示例

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> jsonObject = (Map<String, Object>) envelope.getMessage();
  WikipediaFeedEvent event = new WikipediaFeedEvent(jsonObject);

  try {
    Map<String, Object> parsedJsonObject = parse(event.getRawEvent());

    parsedJsonObject.put("channel", event.getChannel());
    parsedJsonObject.put("source", event.getSource());
    parsedJsonObject.put("time", event.getTime());

    collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-edits"), parsedJsonObject));
  } catch (Exception e) {
    System.err.println("Unable to parse line: " + event);
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) {
  Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null));
  byte[] eventValue = (byte[]) envelope.getMessage();
  if (interceptor.isPresent()) {
    eventValue = interceptor.get().intercept(eventValue);
  }

  EventData eventData = new EventData(eventValue);

  eventData.getProperties().put(PRODUCE_TIMESTAMP, Long.toString(System.currentTimeMillis()));

  if (config.getSendKeyInEventProperties(systemName)) {
    String keyValue = "";
    if (envelope.getKey() != null) {
      keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey())
              : envelope.getKey().toString();
    }
    eventData.getProperties().put(KEY, keyValue);
  }
  return eventData;
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
/**
 * Serialize and send a coordinator stream message.
 * 
 * @param message
 *          The message to send.
 */
public void send(CoordinatorStreamMessage message) {
  log.debug("Sending {}", message);
  try {
    String source = message.getSource();
    byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray()));
    byte[] value = null;
    if (!message.isDelete()) {
      value = messageSerde.toBytes(message.getMessageMap());
    }
    OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value);
    systemProducer.send(source, envelope);
  } catch (Exception e) {
    throw new SamzaException(e);
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Test
public void testEndToEndFlatten() throws Exception {
  int numMessages = 20;
  TestAvroSystemFactory.messages.clear();
  Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
  LOG.info(" Class Path : " + RelOptUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
  String sql1 =
      "Insert into testavro.outputTopic select Flatten(array_values) as string_value, id from testavro.COMPLEX1";
  List<String> sqlStmts = Collections.singletonList(sql1);
  staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
  SamzaSqlApplicationRunner runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
  runner.runAndWaitForFinish();

  List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);

  int expectedMessages = 0;
  // Flatten de-normalizes the data. So there is separate record for each entry in the array.
  for (int index = 1; index < numMessages; index++) {
    expectedMessages = expectedMessages + Math.max(1, index);
  }
  Assert.assertEquals(expectedMessages, outMessages.size());
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Test
public void testEndToEndSubQuery() throws Exception {
  int numMessages = 20;
  TestAvroSystemFactory.messages.clear();
  Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
  String sql1 =
      "Insert into testavro.outputTopic select Flatten(a) as id from (select MyTestArray(id) a from testavro.SIMPLE1)";
  List<String> sqlStmts = Collections.singletonList(sql1);
  staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
  SamzaSqlApplicationRunner runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
  runner.runAndWaitForFinish();

  List<OutgoingMessageEnvelope> outMessages = new ArrayList<>(TestAvroSystemFactory.messages);

  int expectedMessages = 0;
  // Flatten de-normalizes the data. So there is separate record for each entry in the array.
  for (int index = 1; index < numMessages; index++) {
    expectedMessages = expectedMessages + Math.max(1, index);
  }
  Assert.assertEquals(expectedMessages, outMessages.size());
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
private BiFunction<IncomingMessageEnvelope, ESPushTaskConfig.ESIndexSpec, OutgoingMessageEnvelope> getOutMsgExtractor(ESPushTaskConfig.ESIndexSpec spec) {
    BiFunction<IncomingMessageEnvelope, ESPushTaskConfig.ESIndexSpec, OutgoingMessageEnvelope> func = null;
    switch (spec.metadataSrc) {
        case KEY_DOC_ID:
            func = this::getSimpleOutMsg;
            break;
        case KEY_AVRO:
            func = this::getAvroKeyOutMsg;
            break;
        case KEY_JSON:
            func = this::getJsonKeyOutMsg;
            break;
        case EMBEDDED:
            func = this::getEmbeddedOutMsg;
            break;
    }
    return func;
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Test
public void testDefaultDocIdWithAvroKeyConfig() throws Exception {
    ESPushTaskConfig.ESIndexSpec esConfig = getEsIndexSpec("key_avro", true);
    ESPushTask task = getEsPushTask();
    ActionRequestKey inKey = ActionRequestKey.newBuilder()
        .setEventTsUnixMs(3L)
        .setPartitionTsUnixMs(4L)
        .setVersionType(VersionType.EXTERNAL)
        .setVersion(5L)
        .build();
    when(task.avroSerde.fromBytes(null)).thenReturn(inKey);
    OutgoingMessageEnvelope out = task.getAvroKeyOutMsg(getInMsg(""), esConfig);
    HTTPBulkLoader.ActionRequest req = (HTTPBulkLoader.ActionRequest) out.getMessage();
    assertEquals("fake-0-1234", req.key.getId().toString());
    assertEquals(Action.INDEX, req.key.getAction());
    assertEquals(4L, req.key.getPartitionTsUnixMs().longValue());
    assertEquals(3L, req.key.getEventTsUnixMs().longValue());
    assertEquals(VersionType.EXTERNAL, req.key.getVersionType());
    assertEquals(5L, req.key.getVersion().longValue());
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Test
public void testDefaultDocIdWithEmbeddedConfig() throws Exception {
    ESPushTaskConfig.ESIndexSpec esConfig = getEsIndexSpec("embedded", true);
    ESPushTask task = getEsPushTask();
    HashMap<String, Object> doc = new HashMap<>();
    when(task.jsonSerde.fromBytes(null)).thenReturn(doc);
    when(task.jsonSerde.toString(doc)).thenReturn("");
    long tsNowMs = 1453952662L;
    OutgoingMessageEnvelope out = task.getEmbeddedOutMsg(getInMsg(null), esConfig, Optional.of(tsNowMs));
    HTTPBulkLoader.ActionRequest req = (HTTPBulkLoader.ActionRequest) out.getMessage();
    assertEquals("fake-0-1234", req.key.getId().toString());
    assertEquals(Action.INDEX, req.key.getAction());
    assertEquals(tsNowMs, req.key.getPartitionTsUnixMs().longValue());
    assertNull("Do not default event time", req.key.getEventTsUnixMs());
    assertNull("Version not set", req.key.getVersion());
    assertNull("Version type not set", req.key.getVersionType());
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public void window(MessageCollector collector,
  TaskCoordinator coordinator) {

  KeyValueIterator<String, String> entries = store.all();
  while (entries.hasNext()) {                                        // c
    Entry<String, String> entry = entries.next();
    String key = entry.getKey();
    String value = entry.getValue();
    if (isTimestampKey(key) && Cart.isAbandoned(value)) {            // d
      String shopper = extractShopper(key);
      String cart = store.get(asCartKey(shopper));
      
      AbandonedCartEvent event =
        new AbandonedCartEvent(shopper, cart);
      collector.send(new OutgoingMessageEnvelope(
        new SystemStream("kafka", "derived-events-ch04"), event));    // e
      
      resetShopper(shopper);
    }
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
private void fanOut(String sender, Map<String, Object> message, MessageCollector collector) {
  // Colon is used as separator, and semicolon is lexicographically after colon
  KeyValueIterator<String, String> followers = socialGraph.range(sender + ":", sender + ";");

  try {
    while (followers.hasNext()) {
      String[] follow = followers.next().getKey().split(":");
      if (!follow[0].equals(sender)) {
        throw new IllegalStateException("Social graph db prefix doesn't match: " + sender + " != " + follow[0]);
      }
      message.put("recipient", follow[1]);
      collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.DELIVERIES_STREAM, follow[1], null, message));
    }
  } finally {
    followers.close();
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  for (int i = 0; i < 100 && messagesSent < NewsfeedConfig.NUM_FOLLOW_EVENTS; i++, messagesSent++) {
    String follower = NewsfeedConfig.randomUser();
    String followee = NewsfeedConfig.randomUser();

    HashMap<String, Object> message = new HashMap<String, Object>();
    message.put("event", "follow");
    message.put("follower", follower);
    message.put("followee", followee);
    message.put("time", NewsfeedConfig.currentDateTime());
    collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.FOLLOWS_STREAM, followee, null, message));
  }

  if (messagesSent % 100000 == 0) {
    log.info("Generated " + messagesSent + " follow events");
  }

  if (messagesSent == NewsfeedConfig.NUM_FOLLOW_EVENTS) {
    log.info("Finished generating random follower graph");
    coordinator.shutdown(RequestScope.CURRENT_TASK);
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public void send(String source, OutgoingMessageEnvelope envelope) {
  String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(),
      new String((byte[]) envelope.getMessage()));
  LOG.info(msg);

  if (envelope.getKey() != null) {
    System.out.println(String.format("Key:%s Value:%s", envelope.getKey(),
        new String((byte[]) envelope.getMessage())));
  } else {
    System.out.println(new String((byte[]) envelope.getMessage()));
  }
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) throws Exception {
  if (msgCount == maxMessages) {
    coordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
  }

  collector.send ( new OutgoingMessageEnvelope(outputStream, ByteBuffer.allocate(4).putInt((Integer)envelope.getMessage() * 2).array()));
  msgCount++;
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  counts.put("edits", edits);
  counts.put("bytes-added", byteDiff);
  counts.put("unique-titles", titles.size());
  counts.put("edits-all-time", store.get("count-edits-all-time"));

  collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-stats"), counts));

  // Reset counts after windowing.
  edits = 0;
  byteDiff = 0;
  titles = new HashSet<String>();
  counts = new HashMap<String, Integer>();
}
 

import org.apache.samza.system.OutgoingMessageEnvelope; //导入依赖的package包/类
@Override
public IndexRequest getIndexRequest(OutgoingMessageEnvelope envelope) {
  IndexRequest indexRequest = getRequest(envelope);

  Optional<String> id = getId(envelope);
  if (id.isPresent()) {
    indexRequest.id(id.get());
  }

  Optional<String> routingKey = getRoutingKey(envelope);
  if (routingKey.isPresent()) {
    indexRequest.routing(routingKey.get());
  }

  Optional<Long> version = getVersion(envelope);
  if (version.isPresent()) {
    indexRequest.version(version.get());
  }

  Optional<VersionType> versionType = getVersionType(envelope);
  if (versionType.isPresent()) {
    indexRequest.versionType(versionType.get());
  }

  setSource(envelope, indexRequest);

  return indexRequest;
}