Java IncomingMessageEnvelope类代码示例

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator coordinator) {
  Map<String, Object> jsonObject = (Map<String, Object>) envelope.getMessage();
  WikipediaFeedEvent event = new WikipediaFeedEvent(jsonObject);

  try {
    Map<String, Object> parsedJsonObject = parse(event.getRawEvent());

    parsedJsonObject.put("channel", event.getChannel());
    parsedJsonObject.put("source", event.getSource());
    parsedJsonObject.put("time", event.getTime());

    collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wikipedia-edits"), parsedJsonObject));
  } catch (Exception e) {
    System.err.println("Unable to parse line: " + event);
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(
  Set<SystemStreamPartition> systemStreamPartitions, long timeout)
  throws InterruptedException {
  systemStreamPartitions.forEach(systemStreamPartition -> {
    Future status = readerRunnableStatus.get(systemStreamPartition);
    if (status.isDone()) {
      try {
        status.get();
      } catch (ExecutionException | InterruptedException e) {
        MultiFileHdfsReader reader = readers.get(systemStreamPartition);
        LOG.warn(
          String.format("Detect failure in ReaderRunnable for ssp: %s. Try to reconnect now.", systemStreamPartition),
          e);
        reader.reconnect();
        readerRunnableStatus.put(systemStreamPartition, executorService.submit(new ReaderRunnable(reader)));
      }
    }
  });
  return super.poll(systemStreamPartitions, timeout);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
@Test(expected = SamzaException.class)
public void testReachingMaxReconnect() {
  int numMaxRetries = 3;
  SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
  MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0", numMaxRetries);
  // first read a few events, and then reconnect
  for (int i = 0; i < NUM_EVENTS / 2; i++) {
    multiReader.readNext();
  }
  for (int i = 0; i < numMaxRetries; i++) {
    IncomingMessageEnvelope envelope = multiReader.readNext();
    multiReader.reconnect();
    IncomingMessageEnvelope envelopeAfterReconnect = multiReader.readNext();
    Assert.assertEquals(envelope, envelopeAfterReconnect);
  }
  multiReader.readNext();
  multiReader.reconnect();
  Assert.fail();
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
private void verifyRecords(List<IncomingMessageEnvelope> outputRecords, List<Record> inputRecords, String shardId) {
  Iterator outputRecordsIter = outputRecords.iterator();
  inputRecords.forEach(record -> {
      IncomingMessageEnvelope envelope = (IncomingMessageEnvelope) outputRecordsIter.next();
      String outputKey = (String) envelope.getKey();
      KinesisIncomingMessageEnvelope kinesisMessageEnvelope = (KinesisIncomingMessageEnvelope) envelope;
      Assert.assertEquals(outputKey, record.getPartitionKey());
      Assert.assertEquals(kinesisMessageEnvelope.getSequenceNumber(), record.getSequenceNumber());
      Assert.assertEquals(kinesisMessageEnvelope.getApproximateArrivalTimestamp(),
          record.getApproximateArrivalTimestamp());
      Assert.assertEquals(kinesisMessageEnvelope.getShardId(), shardId);
      ByteBuffer outputData = ByteBuffer.wrap((byte[]) kinesisMessageEnvelope.getMessage());
      record.getData().rewind();
      Assert.assertTrue(outputData.equals(record.getData()));
      verifyOffset(envelope.getOffset(), record, shardId);
    });
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
@Test(expected = SamzaException.class)
public void testReadFailsOnSerdeExceptions() throws Exception {
  KafkaStreamSpec checkpointSpec = new KafkaStreamSpec(CHECKPOINT_TOPIC, CHECKPOINT_TOPIC,
      CHECKPOINT_SYSTEM, 1);
  Config mockConfig = mock(Config.class);
  when(mockConfig.get(JobConfig.SSP_GROUPER_FACTORY())).thenReturn(GROUPER_FACTORY_CLASS);

  // mock out a consumer that returns a single checkpoint IME
  SystemStreamPartition ssp = new SystemStreamPartition("system-1", "input-topic", new Partition(0));
  List<List<IncomingMessageEnvelope>> checkpointEnvelopes = ImmutableList.of(
      ImmutableList.of(newCheckpointEnvelope(TASK1, ssp, "0")));
  SystemConsumer mockConsumer = newConsumer(checkpointEnvelopes);

  SystemAdmin mockAdmin = newAdmin("0", "1");
  SystemFactory factory = newFactory(mock(SystemProducer.class), mockConsumer, mockAdmin);

  // wire up an exception throwing serde with the checkpointmanager
  KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory,
      true, mockConfig, mock(MetricsRegistry.class), new ExceptionThrowingCheckpointSerde(), new KafkaCheckpointLogKeySerde());
  checkpointManager.register(TASK1);
  checkpointManager.start();

  // expect an exception from ExceptionThrowingSerde
  checkpointManager.readLastCheckpoint(TASK1);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
@Override
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException {
  Throwable handlerError = eventHubHandlerError.get();

  if (handlerError != null) {
    if (isErrorTransient(handlerError)) {
      // Log a warning if the error is transient
      // Partition receiver handler OnError should have handled it by recreating the receiver
      LOG.warn("Received a transient error from event hub partition receiver, restarted receiver", handlerError);
    } else {
      // Propagate the error to user if the throwable is either
      // 1. permanent ServiceBusException error from client
      // 2. SamzaException thrown bu the EventHubConsumer
      //   2a. Interrupted during put operation to BEM
      //   2b. Failure in renewing the Partititon Receiver
      String msg = "Received a non transient error from event hub partition receiver";
      throw new SamzaException(msg, handlerError);
    }
  }

  return super.poll(systemStreamPartitions, timeout);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * returns all unread messages of a specific type, after an iterator on the stream
 *
 * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned
 * @param type     the type of the messages to be returned
 * @return a set of unread messages of a given type, after a given iterator
 */
public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) {
  LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>();
  while (iterator.hasNext()) {
    IncomingMessageEnvelope envelope = iterator.next();
    Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
    Map<String, Object> valueMap = null;
    if (envelope.getMessage() != null) {
      valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
    }
    CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
    if (type == null || type.equals(coordinatorStreamMessage.getType())) {
      messages.add(coordinatorStreamMessage);
    }
  }
  return messages;
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * Passes the incoming message envelopes along to the {@link InputOperatorImpl} node
 * for the input {@link SystemStream}.
 * <p>
 * From then on, each {@link org.apache.samza.operators.impl.OperatorImpl} propagates its transformed output to
 * its chained {@link org.apache.samza.operators.impl.OperatorImpl}s itself.
 *
 * @param ime incoming message envelope to process
 * @param collector the collector to send messages with
 * @param coordinator the coordinator to request commits or shutdown
 */
@Override
public final void process(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) {
  SystemStream systemStream = ime.getSystemStreamPartition().getSystemStream();
  InputOperatorImpl inputOpImpl = operatorImplGraph.getInputOperator(systemStream);
  if (inputOpImpl != null) {
    switch (MessageType.of(ime.getMessage())) {
      case USER_MESSAGE:
        inputOpImpl.onMessage(KV.of(ime.getKey(), ime.getMessage()), collector, coordinator);
        break;

      case END_OF_STREAM:
        EndOfStreamMessage eosMessage = (EndOfStreamMessage) ime.getMessage();
        inputOpImpl.aggregateEndOfStream(eosMessage, ime.getSystemStreamPartition(), collector, coordinator);
        break;

      case WATERMARK:
        WatermarkMessage watermarkMessage = (WatermarkMessage) ime.getMessage();
        inputOpImpl.aggregateWatermark(watermarkMessage, ime.getSystemStreamPartition(), collector, coordinator);
        break;
    }
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
public TaskCallbackImpl createCallback(TaskName taskName,
    IncomingMessageEnvelope envelope,
    ReadableCoordinator coordinator) {
  final TaskCallbackImpl callback = new TaskCallbackImpl(listener, taskName, envelope, coordinator, seqNum++, clock.nanoTime());
  if (timer != null) {
    Runnable timerTask = new Runnable() {
      @Override
      public void run() {
        String msg = "Task " + callback.taskName + " callback times out";
        callback.failure(new TaskCallbackTimeoutException(msg));
      }
    };
    ScheduledFuture scheduledFuture = timer.schedule(timerTask, timeout, TimeUnit.MILLISECONDS);
    callback.setScheduledFuture(scheduledFuture);
  }

  return callback;
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * Block the runloop thread if all tasks are busy. When a task worker finishes or window/commit completes,
 * it will resume the runloop.
 */
private void blockIfBusy(IncomingMessageEnvelope envelope) {
  synchronized (latch) {
    while (!shutdownNow && throwable == null) {
      for (AsyncTaskWorker worker : taskWorkers) {
        if (worker.state.isReady()) {
          // should continue running if any worker state is ready
          // consumerMultiplexer will block on polling for empty partitions so it won't cause busy loop
          return;
        }
      }

      try {
        log.trace("Block loop thread");
        latch.wait();
      } catch (InterruptedException e) {
        throw new SamzaException("Run loop is interrupted", e);
      }
    }
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * Process asynchronously. The callback needs to be fired once the processing is done.
 */
private void process() {
  final IncomingMessageEnvelope envelope = state.fetchEnvelope();
  log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset());

  final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName());
  TaskCallbackFactory callbackFactory = new TaskCallbackFactory() {
    @Override
    public TaskCallback createCallback() {
      state.startProcess();
      containerMetrics.processes().inc();
      return callbackManager.createCallback(task.taskName(), envelope, coordinator);
    }
  };

  task.process(envelope, coordinator, callbackFactory);
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
/**
 * Fetch the pending envelope in the pending queue for the task to process.
 * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop
 * will be able to choose new messages from this SSP for the task to process. Note that we
 * update only when the envelope is first time being processed. This solves the issue in
 * Broadcast stream where a message need to be processed by multiple tasks. In that case,
 * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates
 * the chooser with the next envelope in the broadcast stream partition.
 * The function will be called in the run loop thread so no synchronization.
 * @return
 */
private IncomingMessageEnvelope fetchEnvelope() {
  PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove();
  int queueSize = pendingEnvelopeQueue.size();
  taskMetrics.pendingMessages().set(queueSize);
  log.trace("fetch envelope ssp {} offset {} to process.",
      pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset());
  log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize);

  if (pendingEnvelope.markProcessed()) {
    SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition();
    consumerMultiplexer.tryUpdate(partition);
    log.debug("Update chooser for {}", partition);
  }
  return pendingEnvelope.envelope;
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
@Override
public void processAsync(final IncomingMessageEnvelope envelope,
    final MessageCollector collector,
    final TaskCoordinator coordinator,
    final TaskCallback callback) {
  if (executor != null) {
    executor.submit(new Runnable() {
      @Override
      public void run() {
        process(envelope, collector, coordinator, callback);
      }
    });
  } else {
    // legacy mode: running all tasks in the runloop thread
    process(envelope, collector, coordinator, callback);
  }
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException {
  Map<SystemStreamPartition, List<IncomingMessageEnvelope>> map = new LinkedHashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>();
  assertEquals(1, systemStreamPartitions.size());
  SystemStreamPartition systemStreamPartition = systemStreamPartitions.iterator().next();
  assertEquals(expectedSystemStreamPartition, systemStreamPartition);

  if (pollCount++ == 0) {
    List<IncomingMessageEnvelope> list = new ArrayList<IncomingMessageEnvelope>();
    SetConfig setConfig1 = new SetConfig("test", "job.name", "my-job-name");
    SetConfig setConfig2 = new SetConfig("test", "job.id", "1234");
    Delete delete = new Delete("test", "job.name", SetConfig.TYPE);
    list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig1.getKeyArray()), serialize(setConfig1.getMessageMap())));
    list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig2.getKeyArray()), serialize(setConfig2.getMessageMap())));
    list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(delete.getKeyArray()), delete.getMessageMap()));
    map.put(systemStreamPartition, list);
  }

  return map;
}
 

import org.apache.samza.system.IncomingMessageEnvelope; //导入依赖的package包/类
private void pollingEntranceProcessor() throws InterruptedException {
  int messageCnt = 0;
  while (!this.entranceProcessor.isFinished()) {
    messageCnt = this.getNumMessagesInQueue(systemStreamPartition);
    if (this.entranceProcessor.hasNext() && messageCnt < 10000) { // soft
                                                                  // limit
                                                                  // on the
                                                                  // size of
                                                                  // the
                                                                  // queue
      this.put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, null, null,
          this.entranceProcessor.nextEvent()));
    } else {
      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
        break;
      }
    }
  }

  // Send last event
  this.put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, null, null,
      this.entranceProcessor.nextEvent()));
}