Java IncomingMessageEnvelope.getMessage方法代码示例

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> jsonObject = (Map<String, Object>) envelope.getMessage();
  WikipediaFeedEvent event = new WikipediaFeedEvent(jsonObject);

  try {
    Map<String, Object> parsedJsonObject = parse(event.getRawEvent());

    parsedJsonObject.put("channel", event.getChannel());
    parsedJsonObject.put("source", event.getSource());
    parsedJsonObject.put("time", event.getTime());

    collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-edits"), parsedJsonObject));
  } catch (Exception e) {
    System.err.println("Unable to parse line: " + event);
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
/**
 * returns all unread messages of a specific type, after an iterator on the stream
 *
 * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned
 * @param type     the type of the messages to be returned
 * @return a set of unread messages of a given type, after a given iterator
 */
public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) {
  LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>();
  while (iterator.hasNext()) {
    IncomingMessageEnvelope envelope = iterator.next();
    Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
    Map<String, Object> valueMap = null;
    if (envelope.getMessage() != null) {
      valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
    }
    CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
    if (type == null || type.equals(coordinatorStreamMessage.getType())) {
      messages.add(coordinatorStreamMessage);
    }
  }
  return messages;
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
/**
 * Passes the incoming message envelopes along to the {@link InputOperatorImpl} node
 * for the input {@link SystemStream}.
 * <p>
 * From then on, each {@link org.apache.samza.operators.impl.OperatorImpl} propagates its transformed output to
 * its chained {@link org.apache.samza.operators.impl.OperatorImpl}s itself.
 *
 * @param ime incoming message envelope to process
 * @param collector the collector to send messages with
 * @param coordinator the coordinator to request commits or shutdown
 */
@Override
public final void process(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) {
  SystemStream systemStream = ime.getSystemStreamPartition().getSystemStream();
  InputOperatorImpl inputOpImpl = operatorImplGraph.getInputOperator(systemStream);
  if (inputOpImpl != null) {
    switch (MessageType.of(ime.getMessage())) {
      case USER_MESSAGE:
        inputOpImpl.onMessage(KV.of(ime.getKey(), ime.getMessage()), collector, coordinator);
        break;

      case END_OF_STREAM:
        EndOfStreamMessage eosMessage = (EndOfStreamMessage) ime.getMessage();
        inputOpImpl.aggregateEndOfStream(eosMessage, ime.getSystemStreamPartition(), collector, coordinator);
        break;

      case WATERMARK:
        WatermarkMessage watermarkMessage = (WatermarkMessage) ime.getMessage();
        inputOpImpl.aggregateWatermark(watermarkMessage, ime.getSystemStreamPartition(), collector, coordinator);
        break;
    }
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) throws Exception {
    String stream = envelope.getSystemStreamPartition().getSystemStream().getStream();
    Object message = envelope.getMessage();

    List<Processor> processors = Processor.getProcessors(stream, this.config, this.context, this.storeManager);
    if (message instanceof Map) {
        for (Processor processor : processors) {
            processor.process((Map<String, Object>) message, collector);
        }
    } else {
        log.warn("This message is not a map class: " + message);
    }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> edit = (Map<String, Object>) envelope.getMessage();
  Map<String, Boolean> flags = (Map<String, Boolean>) edit.get("flags");

  Integer editsAllTime = store.get("count-edits-all-time");
  if (editsAllTime == null) editsAllTime = 0;
  store.put("count-edits-all-time", editsAllTime + 1);

  edits += 1;
  titles.add((String) edit.get("title"));
  byteDiff += (Integer) edit.get("diff-bytes");

  for (Map.Entry<String, Boolean> flag : flags.entrySet()) {
    if (Boolean.TRUE.equals(flag.getValue())) {
      Integer count = counts.get(flag.getKey());

      if (count == null) {
        count = 0;
      }

      count += 1;
      counts.put(flag.getKey(), count);
    }
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
public IncomingMessageEnvelope readNext() {
  if (!hasNext()) {
    LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition);
    return null;
  }
  // record the next offset before we read, so when the read fails and we reconnect,
  // we seek to the same offset that we try below
  curSingleFileOffset = curReader.nextOffset();
  IncomingMessageEnvelope messageEnvelope = curReader.readNext();
  // Copy everything except for the offset. Turn the single-file style offset into a multi-file one
  return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(),
    messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize());
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Test
public void testRandomRead() throws Exception {
  SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
  SingleFileHdfsReader reader = new AvroFileHdfsReader(ssp);
  reader.open(AVRO_FILE, "0");
  for (int i = 0;i < NUM_EVENTS / 2; i++) {
    reader.readNext();
  }
  String offset = reader.nextOffset();
  IncomingMessageEnvelope envelope = reader.readNext();
  Assert.assertEquals(offset, envelope.getOffset());

  GenericRecord record1 = (GenericRecord) envelope.getMessage();

  for (int i = 0; i < 5; i++) reader.readNext();

  // seek to the offset within the same reader
  reader.seek(offset);
  Assert.assertEquals(offset, reader.nextOffset());
  envelope = reader.readNext();
  Assert.assertEquals(offset, envelope.getOffset());
  GenericRecord record2 = (GenericRecord) envelope.getMessage();
  Assert.assertEquals(record1, record2);
  reader.close();

  // open a new reader and initialize it with the offset
  reader = new AvroFileHdfsReader(ssp);
  reader.open(AVRO_FILE, offset);
  envelope = reader.readNext();
  Assert.assertEquals(offset, envelope.getOffset());
  GenericRecord record3 = (GenericRecord) envelope.getMessage();
  Assert.assertEquals(record1, record3);
  reader.close();
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  String key = (String) envelope.getKey();
  String value = (String) envelope.getMessage();
  String[] pieces = value.split("-");
  int epoch = Integer.parseInt(pieces[0]);

  int partition = Integer.parseInt(pieces[1].split(" ")[1]);
  Partitions partitions = loadPartitions(epoch, key);
  logger.info("Joiner got epoch = " + epoch + ", partition = " + partition + ", parts = " + partitions);
  if (partitions.epoch < epoch) {
    // we are in a new era
    if (partitions.partitions.size() != expected)
      throw new IllegalArgumentException("Should have " + expected + " partitions when new epoch starts.");
    logger.info("Reseting epoch to " + epoch);
    this.store.delete(key);
    partitions.epoch = epoch;
    partitions.partitions.clear();
    partitions.partitions.add(partition);
  } else if (partitions.epoch > epoch) {
    logger.info("Ignoring message for epoch " + epoch);
  } else {
    partitions.partitions.add(partition);
    if (partitions.partitions.size() == expected) {
      logger.info("Completed: " + key + " -> " + Integer.toString(epoch));
      collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "completed-keys"), key, Integer.toString(epoch)));
    }
  }
  this.store.put(key, partitions.toString());
  logger.info("Join store in Task " + this.taskName + " " + key + " -> " + partitions.toString());
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  String key = (String) envelope.getKey();
  String epoch = (String) envelope.getMessage();
  logger.info("Got key=" + key + ", epoch = " + epoch + " in checker...");
  checkEpoch(epoch);
  this.store.put(key, epoch);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  messagesProcessed += 1;
  String input = (String) envelope.getMessage();
  Integer number = Integer.valueOf(input);
  Integer output = number.intValue() * -1;
  collector.send(new OutgoingMessageEnvelope(outputSystemStream, output.toString()));
  if (messagesProcessed >= maxMessages) {
    coordinator.shutdown(RequestScope.ALL_TASKS_IN_CONTAINER);
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector,
    TaskCoordinator taskCoordinator)
    throws Exception {

  Object message = incomingMessageEnvelope.getMessage();

  String key = new String((byte[]) incomingMessageEnvelope.getKey());
  Integer val = Integer.valueOf((String) message);

  LOG.info("Stream processor " + processorId + ";key=" + key + ";offset=" + incomingMessageEnvelope.getOffset()
      + "; totalRcvd=" + processedMessageCount + ";val=" + val + "; ssp=" + incomingMessageEnvelope
      .getSystemStreamPartition());

  // inject a failure
  if (val >= BAD_MESSAGE_KEY && processorId.equals(processorIdToFail)) {
    LOG.info("process method failing for msg=" + message);
    throw new Exception("Processing in the processor " + processorId + " failed ");
  }

  messageCollector.send(new OutgoingMessageEnvelope(new SystemStream(outputSystem, outputTopic), message));
  processedMessageCount++;

  synchronized (endLatch) {
    if (Integer.valueOf(key) < BAD_MESSAGE_KEY) {
      endLatch.countDown();
    }
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator)
    throws Exception {
  SamzaStream output = (SamzaStream) this.getOutputStream();
  if (output == null)
    return; // if there is no output stream, do nothing
  output.setCollector(collector);
  ContentEvent event = (ContentEvent) envelope.getMessage();
  output.put(event);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
                    TaskCoordinator coordinator) throws Exception {

    GenericRecord genericRecord = (GenericRecord) envelope.getMessage();

    FortuneRequest request = (FortuneRequest) SpecificData.get().deepCopy(FortuneRequest.SCHEMA$, genericRecord);

    String fortune = "Unknown";
    double certainty = 0.0;
    if (request.getFirstName().equals("Ragnar")) {
        fortune = "The sons of Ragnar Lothbrok will be spoken of as long as men have tongues to speak.";
        certainty = 1.0;
    }
    else if (request.getFirstName().toString().equals("Lagartha")) {
        if (request.getQuestion().toString().startsWith("Will")) {
            fortune = "I cannot see another child no matter how far i look";
            certainty = 0.5;
        }
        else {
            fortune = "I see a harvest celebrated in blood. I see a trickster whose weapon cleaves you. I see a city made of marble and a burning, broiling ocean.";
            certainty = 0.9;
        }
    }

    Headers header = Headers.newBuilder()
            .setTimestamp(request.header.getTimestamp())
            .build();
    Fortune fortuneRec = Fortune.newBuilder()
            .setHeader(header)
            .setAnswer(fortune)
            .setFirstName(request.getFirstName())
            .setLastName(request.getLastName())
            .setQuestion(request.getQuestion())
            .setDegreeOfCertainty(certainty)
            .build();
    collector.send(new OutgoingMessageEnvelope(outStream, fortuneRec));
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
    String key = (String) envelope.getKey();
    Integer value = (Integer) envelope.getMessage();
    Integer newValue = value * 10;
    collector.send(new OutgoingMessageEnvelope(outputStream, key, newValue));
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
    String key = (String) envelope.getKey();
    Integer value = (Integer) envelope.getMessage();
    counterList.add(new AbstractMap.SimpleEntry<String, Integer>(key, value));
}