Java ByteBufferMessageSet.iterator方法代码示例

import kafka.javaapi.message.ByteBufferMessageSet; //导入方法依赖的package包/类
/**
 * Creates an Iterator of FetchedMessage based on the given message set. The iterator would also updates
 * the offset while iterating.
 */
private Iterator<FetchedMessage> createFetchedMessages(ByteBufferMessageSet messageSet, final AtomicLong offset) {
  final Iterator<MessageAndOffset> messages = messageSet.iterator();
  return new AbstractIterator<FetchedMessage>() {
    @Override
    protected FetchedMessage computeNext() {
      while (messages.hasNext()) {
        MessageAndOffset message = messages.next();
        long msgOffset = message.offset();
        if (msgOffset < offset.get()) {
          LOG.trace("Received old offset {}, expecting {} on {}. Message Ignored.",
                    msgOffset, offset.get(), topicPart);
          continue;
        }

        fetchedMessage.setPayload(message.message().payload());
        fetchedMessage.setOffset(message.offset());
        fetchedMessage.setNextOffset(message.nextOffset());

        return fetchedMessage;
      }
      return endOfData();
    }
  };
}
 

import kafka.javaapi.message.ByteBufferMessageSet; //导入方法依赖的package包/类
@Override
  public void run()
  {
    long offset = 0;
    while (isAlive) {
      // create a fetch request for topic “topic1”, partition 0, current offset, and fetch size of 1MB
      FetchRequest fetchRequest = new FetchRequestBuilder().clientId("default_client").addFetch("topic1", 1, offset, 1000000).build();

//      FetchRequest fetchRequest = new FetchRequest("topic1", 0, offset, 1000000);

      // get the message set from the consumer and print them out
      ByteBufferMessageSet messages = consumer.fetch(fetchRequest).messageSet("topic1", 1);
      Iterator<MessageAndOffset> itr = messages.iterator();

      while (itr.hasNext() && isAlive) {
        MessageAndOffset msg = itr.next();
        // advance the offset after consuming each message
        offset = msg.offset();
        logger.debug("consumed: {} offset: {}", byteBufferToString(msg.message().payload()).toString(), offset);
        receiveCount++;
      }
    }
  }
 

import kafka.javaapi.message.ByteBufferMessageSet; //导入方法依赖的package包/类
/**
 * Creates an Iterator of FetchedMessage based on the given message set. The iterator would also updates
 * the offset while iterating.
 */
private Iterator<FetchedMessage> createFetchedMessages(ByteBufferMessageSet messageSet, final AtomicLong offset) {
  final Iterator<MessageAndOffset> messages = messageSet.iterator();
  return new AbstractIterator<FetchedMessage>() {
    @Override
    protected FetchedMessage computeNext() {
      while (messages.hasNext()) {
        MessageAndOffset message = messages.next();
        long msgOffset = message.offset();
        if (msgOffset < offset.get()) {
          LOG.trace("Received old offset {}, expecting {} on {}. Message Ignored.",
                    msgOffset, offset.get(), topicPart);
          continue;
        }

        offset.set(message.nextOffset());
        fetchedMessage.setPayload(message.message().payload());
        fetchedMessage.setNextOffset(offset.get());

        return fetchedMessage;
      }
      return endOfData();
    }
  };
}
 

import kafka.javaapi.message.ByteBufferMessageSet; //导入方法依赖的package包/类
public boolean getNext(KafkaETLKey key, BytesWritable value) throws IOException {
    if ( !hasMore() ) return false;
    
    boolean gotNext = get(key, value);

    if(_response != null) {

        while ( !gotNext && _respIterator.hasNext()) {
            ByteBufferMessageSet msgSet = _respIterator.next();
            if ( hasError(msgSet)) return false;
            _messageIt = msgSet.iterator();
            gotNext = get(key, value);
        }
    }
    return gotNext;
}
 

import kafka.javaapi.message.ByteBufferMessageSet; //导入方法依赖的package包/类
private Iterator<MessageAndOffset> getIteratorFromFetchResponse(FetchResponse fetchResponse,
    KafkaPartition partition) {
  try {
    ByteBufferMessageSet messageBuffer = fetchResponse.messageSet(partition.getTopicName(), partition.getId());
    return messageBuffer.iterator();
  } catch (Exception e) {
    LOG.warn(String.format("Failed to retrieve next message buffer for partition %s: %s."
        + "The remainder of this partition will be skipped.", partition, e));
    return null;
  }
}