Java MessageCollector.send方法代码示例

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> jsonObject = (Map<String, Object>) envelope.getMessage();
  WikipediaFeedEvent event = new WikipediaFeedEvent(jsonObject);

  try {
    Map<String, Object> parsedJsonObject = parse(event.getRawEvent());

    parsedJsonObject.put("channel", event.getChannel());
    parsedJsonObject.put("source", event.getSource());
    parsedJsonObject.put("time", event.getTime());

    collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-edits"), parsedJsonObject));
  } catch (Exception e) {
    System.err.println("Unable to parse line: " + event);
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void window(MessageCollector collector,
  TaskCoordinator coordinator) {

  KeyValueIterator<String, String> entries = store.all();
  while (entries.hasNext()) {                                        // c
    Entry<String, String> entry = entries.next();
    String key = entry.getKey();
    String value = entry.getValue();
    if (isTimestampKey(key) && Cart.isAbandoned(value)) {            // d
      String shopper = extractShopper(key);
      String cart = store.get(asCartKey(shopper));
      
      AbandonedCartEvent event =
        new AbandonedCartEvent(shopper, cart);
      collector.send(new OutgoingMessageEnvelope(
        new SystemStream("kafka", "derived-events-ch04"), event));    // e
      
      resetShopper(shopper);
    }
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
private void fanOut(String sender, Map<String, Object> message, MessageCollector collector) {
  // Colon is used as separator, and semicolon is lexicographically after colon
  KeyValueIterator<String, String> followers = socialGraph.range(sender + ":", sender + ";");

  try {
    while (followers.hasNext()) {
      String[] follow = followers.next().getKey().split(":");
      if (!follow[0].equals(sender)) {
        throw new IllegalStateException("Social graph db prefix doesn't match: " + sender + " != " + follow[0]);
      }
      message.put("recipient", follow[1]);
      collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.DELIVERIES_STREAM, follow[1], null, message));
    }
  } finally {
    followers.close();
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  for (int i = 0; i < 100 && messagesSent < NewsfeedConfig.NUM_FOLLOW_EVENTS; i++, messagesSent++) {
    String follower = NewsfeedConfig.randomUser();
    String followee = NewsfeedConfig.randomUser();

    HashMap<String, Object> message = new HashMap<String, Object>();
    message.put("event", "follow");
    message.put("follower", follower);
    message.put("followee", followee);
    message.put("time", NewsfeedConfig.currentDateTime());
    collector.send(new OutgoingMessageEnvelope(NewsfeedConfig.FOLLOWS_STREAM, followee, null, message));
  }

  if (messagesSent % 100000 == 0) {
    log.info("Generated " + messagesSent + " follow events");
  }

  if (messagesSent == NewsfeedConfig.NUM_FOLLOW_EVENTS) {
    log.info("Finished generating random follower graph");
    coordinator.shutdown(RequestScope.CURRENT_TASK);
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  counts.put("edits", edits);
  counts.put("bytes-added", byteDiff);
  counts.put("unique-titles", titles.size());
  counts.put("edits-all-time", store.get("count-edits-all-time"));

  collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-stats"), counts));

  // Reset counts after windowing.
  edits = 0;
  byteDiff = 0;
  titles = new HashSet<String>();
  counts = new HashMap<String, Integer>();
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public Collection<Void> handleMessage(M message, MessageCollector collector,
    TaskCoordinator coordinator) {
  Object key, value;
  if (outputStream.isKeyed()) {
    key = ((KV) message).getKey();
    value = ((KV) message).getValue();
  } else {
    key = null;
    value = message;
  }

  collector.send(new OutgoingMessageEnvelope(systemStream, null, key, value));
  return Collections.emptyList();
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) {
  SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(systemStream, true);
  int partitionCount = metadata.getSystemStreamPartitionMetadata().size();
  LOG.info(String.format("Broadcast %s message from task %s to %s with %s partition",
      MessageType.of(message).name(), message.getTaskName(), systemStream, partitionCount));

  for (int i = 0; i < partitionCount; i++) {
    OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message);
    collector.send(envelopeOut);
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public Collection<Void> handleMessage(M message, MessageCollector collector,
    TaskCoordinator coordinator) {
  K key = keyFunction.apply(message);
  V value = valueFunction.apply(message);
  Long partitionKey = key == null ? 0L : null;
  collector.send(new OutgoingMessageEnvelope(systemStream, partitionKey, key, value));
  return Collections.emptyList();
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  Integer epoch = getInt(EPOCH);
  if (epoch == null) {
    resetEpoch();
    return;
  }
  int counter = getInt(COUNT);
  if (counter < max) {
    logger.info("Emitting: " + counter + ", epoch = " + epoch + ", task = " + taskName);
    OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(new SystemStream("kafka", "emitted"), Integer.toString(counter), epoch + "-" + taskName.toString());
    collector.send(envelope);
    this.state.put(COUNT, Integer.toString(getInt(COUNT) + 1));
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  String key = (String) envelope.getKey();
  String value = (String) envelope.getMessage();
  String[] pieces = value.split("-");
  int epoch = Integer.parseInt(pieces[0]);

  int partition = Integer.parseInt(pieces[1].split(" ")[1]);
  Partitions partitions = loadPartitions(epoch, key);
  logger.info("Joiner got epoch = " + epoch + ", partition = " + partition + ", parts = " + partitions);
  if (partitions.epoch < epoch) {
    // we are in a new era
    if (partitions.partitions.size() != expected)
      throw new IllegalArgumentException("Should have " + expected + " partitions when new epoch starts.");
    logger.info("Reseting epoch to " + epoch);
    this.store.delete(key);
    partitions.epoch = epoch;
    partitions.partitions.clear();
    partitions.partitions.add(partition);
  } else if (partitions.epoch > epoch) {
    logger.info("Ignoring message for epoch " + epoch);
  } else {
    partitions.partitions.add(partition);
    if (partitions.partitions.size() == expected) {
      logger.info("Completed: " + key + " -> " + Integer.toString(epoch));
      collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "completed-keys"), key, Integer.toString(epoch)));
    }
  }
  this.store.put(key, partitions.toString());
  logger.info("Join store in Task " + this.taskName + " " + key + " -> " + partitions.toString());
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void window(MessageCollector collector, TaskCoordinator coordinator) {
  String currentEpoch = this.store.get(CURRENT_EPOCH);
  logger.info("Checking if epoch " + currentEpoch + " is complete.");
  int count = 0;
  KeyValueIterator<String, String> iter = this.store.all();

  while (iter.hasNext()) {
    Entry<String, String> entry = iter.next();
    String foundEpoch = entry.getValue();
    if (foundEpoch.equals(currentEpoch)) {
      count += 1;
    } else {
      logger.info("####### Found a different epoch! - " + foundEpoch + " Current epoch is " + currentEpoch);
    }
  }
  iter.close();
  if (count == expectedKeys + 1) {
    logger.info("Epoch " + currentEpoch + " is complete.");
    int nextEpoch = Integer.parseInt(currentEpoch) + 1;
    for (int i = 0; i < numPartitions; i++) {
      logger.info("Emitting next epoch - " + Integer.toString(i) + " -> " + Integer.toString(nextEpoch));
      collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "epoch"), Integer.toString(i), Integer.toString(nextEpoch)));
    }
    this.store.put(CURRENT_EPOCH, Integer.toString(nextEpoch));
  } else if (count > expectedKeys + 1) {
    throw new IllegalStateException("Got " + count + " keys, which is more than the expected " + (expectedKeys + 1));
  } else {
    logger.info("Only found " + count + " valid keys, try again later.");
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  messagesProcessed += 1;
  String input = (String) envelope.getMessage();
  Integer number = Integer.valueOf(input);
  Integer output = number.intValue() * -1;
  collector.send(new OutgoingMessageEnvelope(outputSystemStream, output.toString()));
  if (messagesProcessed >= maxMessages) {
    coordinator.shutdown(RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector,
    TaskCoordinator taskCoordinator)
    throws Exception {

  Object message = incomingMessageEnvelope.getMessage();

  String key = new String((byte[]) incomingMessageEnvelope.getKey());
  Integer val = Integer.valueOf((String) message);

  LOG.info("Stream processor " + processorId + ";key=" + key + ";offset=" + incomingMessageEnvelope.getOffset()
      + "; totalRcvd=" + processedMessageCount + ";val=" + val + "; ssp=" + incomingMessageEnvelope
      .getSystemStreamPartition());

  // inject a failure
  if (val >= BAD_MESSAGE_KEY && processorId.equals(processorIdToFail)) {
    LOG.info("process method failing for msg=" + message);
    throw new Exception("Processing in the processor " + processorId + " failed ");
  }

  messageCollector.send(new OutgoingMessageEnvelope(new SystemStream(outputSystem, outputTopic), message));
  processedMessageCount++;

  synchronized (endLatch) {
    if (Integer.valueOf(key) < BAD_MESSAGE_KEY) {
      endLatch.countDown();
    }
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@Override
public void process(
    IncomingMessageEnvelope incomingMessageEnvelope,
    MessageCollector messageCollector,
    TaskCoordinator taskCoordinator) throws Exception {
  messageCollector.send(
      new OutgoingMessageEnvelope(
          new SystemStream(outputSystem, outputTopic),
          incomingMessageEnvelope.getMessage()));
  processedMessageCount++;
  if (processedMessageCount == expectedMessageCount) {
    taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

import org.apache.samza.task.MessageCollector; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
    String key = (String) envelope.getKey();
    Integer value = (Integer) envelope.getMessage();
    Integer newValue = value * 10;
    collector.send(new OutgoingMessageEnvelope(outputStream, key, newValue));
}