Java OffsetMetadataAndError.offset方法代码示例

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
public long readOffset(String topic, int partition, String groupId) {
    BlockingChannel channel = new BlockingChannel(host, port,
            BlockingChannel.UseDefaultBufferSize(),
            BlockingChannel.UseDefaultBufferSize(),
            (int) Duration.ofSeconds(10).toMillis());
    try {
        channel.connect();
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(
                groupId, singletonList(topicAndPartition), (short) 1, RANDOM.nextInt(), "test-checker");
        channel.send(offsetFetchRequest.underlying());
        OffsetFetchResponse fetchResponse = OffsetFetchResponse.readFrom(channel.receive().payload());
        OffsetMetadataAndError result = fetchResponse.offsets().get(topicAndPartition);
        return result.offset();
    } finally {
        channel.disconnect();
    }
}
 

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
/**
 * 从保存consumer消费者offset偏移量的位置获取当前consumer对应的偏移量
 *
 * @param consumer    消费者
 * @param groupId     Group Id
 * @param clientName  client名称
 * @param topic       topic名称
 * @param partitionID 分区id
 *
 * @return
 */
public long getOffsetOfTopicAndPartition(SimpleConsumer consumer, String groupId, String clientName, String
        topic, int partitionID) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionID);
    List<TopicAndPartition> requestInfo = new ArrayList<TopicAndPartition>();
    requestInfo.add(topicAndPartition);
    OffsetFetchRequest request = new OffsetFetchRequest(groupId, requestInfo, 0, clientName);
    OffsetFetchResponse response = consumer.fetchOffsets(request);

    // 获取返回值
    Map<TopicAndPartition, OffsetMetadataAndError> returnOffsetMetadata = response.offsets();
    // 处理返回值
    if (returnOffsetMetadata != null && !returnOffsetMetadata.isEmpty()) {
        // 获取当前分区对应的偏移量信息
        OffsetMetadataAndError offset = returnOffsetMetadata.get(topicAndPartition);
        if (offset.error().code() == ErrorMapping.NoError()) {
            // 没有异常，表示是正常的，获取偏移量
            return offset.offset();
        } else {
            // 当Consumer第一次连接的时候(zk中不在当前topic对应数据的时候)，会产生UnknownTopicOrPartitionCode异常
            System.out.println("Error fetching data Offset Data the Topic and Partition. Reason: " + offset.error
                    ());
        }
    }

    // 所有异常情况直接返回0
    return 0;
}
 

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
private long fetchNextOffset(SimpleConsumer consumer, String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	List<TopicAndPartition> requestInfo = new ArrayList<>();
	requestInfo.add(topicAndPartition);
	OffsetFetchRequest fetchRequest = new OffsetFetchRequest(groupid, requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), correlationId, clientName);
	OffsetFetchResponse response = null;
	while (true) {
		try {
			logger.debug("partition {} fetch offest request", partition);
			response = consumer.fetchOffsets(fetchRequest);
			if (response != null)
				break;
		} catch (Exception e) {
			logger.error("some error occur when fetch messages", e);
			try {
				Thread.sleep(EXCEPTION_SLEEP_TIME);
			} catch (InterruptedException e1) {
				e1.printStackTrace();
			}
		}
	}

	OffsetMetadataAndError offset = response.offsets().get(topicAndPartition);
	if (offset.error() == 0)
		return offset.offset();
	else
		return 0;
}
 

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
public long fetchOffset(final TopicAndPartition topicAndPartition) throws ConsumerOffsetsException {
    BlockingChannel channel = getConnectionToOffsetManager(groupId);

    final List<TopicAndPartition> partitions = new ArrayList<>();
    partitions.add(topicAndPartition);
    final OffsetFetchRequest fetchRequest = new OffsetFetchRequest(
            groupId,
            partitions,
            VERSION_ID,
            correlationId.incrementAndGet(),
            CLIENT_NAME + "_" + topicAndPartition.topic() + "_" + topicAndPartition.partition());
    try {
        channel.send(fetchRequest.underlying());
        final OffsetFetchResponse fetchResponse = OffsetFetchResponse.readFrom(channel.receive().buffer());
        final OffsetMetadataAndError result = fetchResponse.offsets().get(topicAndPartition);
        final short offsetFetchErrorCode = result.error();
        if (offsetFetchErrorCode == ErrorMapping.NotCoordinatorForConsumerCode()) {
            throw new ConsumerOffsetsException("Not coordinator for consumer - Retry the offset fetch", offsetFetchErrorCode);
        } else if (offsetFetchErrorCode == ErrorMapping.OffsetsLoadInProgressCode()) {
            throw new ConsumerOffsetsException("Offset load in progress - Retry the offset fetch", offsetFetchErrorCode);
        } else {
            // Success
            return result.offset();
            //String retrievedMetadata = result.metadata();
        }
    } catch (ConsumerOffsetsException e) {
        throw e;
    } catch (Throwable t) {
        throw new ConsumerOffsetsException(t);
        // Client should retry the commit
    } finally {
        if (channel != null && channel.isConnected()) {
            channel.disconnect();
        }
    }
}
 

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
long getLastCommitOffset() {
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionMetadata.partitionId());
  List<TopicAndPartition> topicAndPartitions = new ArrayList<>();
  topicAndPartitions.add(topicAndPartition);
  OffsetFetchRequest oRequest = new OffsetFetchRequest(name, topicAndPartitions, (short) 0, 0, name);
  OffsetFetchResponse oResponse = consumer.fetchOffsets(oRequest);
  Map<TopicAndPartition, OffsetMetadataAndError> offsets = oResponse.offsets();
  OffsetMetadataAndError offset = offsets.get(topicAndPartition);
  long currOffset = offset.offset() ;
  if(currOffset < 0) currOffset = 0;
  return currOffset;
}
 

import kafka.common.OffsetMetadataAndError; //导入方法依赖的package包/类
/**
 * Retrieves latest committed offset for a given topic & partition. Uses the
 * new Kafka offset storage API introduced in 0.8.1.
 * @param consumer consumer client to use for request
 * @param topic topic id for which to lookup offset
 * @param partition partition id for which to lookup offset
 * @param clientName client id to include in request
 * @return the offset returned from the lead broker
 */
private static long getLastOffset(final SimpleConsumer consumer, final String topic,
                                  final int partition, final String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);

    List<TopicAndPartition> partitions = new ArrayList<>();
    partitions.add(topicAndPartition);
    OffsetFetchRequest fetchRequest = new OffsetFetchRequest(
            GROUP_ID,
            partitions,
            (short) 1,
            CORRELATION_ID,
            clientName);
    OffsetFetchResponse response = consumer.fetchOffsets(fetchRequest);

    if (response == null) {
        log.error("Error fetching offset data from the Broker.");
        return -1;
    }

    OffsetMetadataAndError result = response.offsets().get(topicAndPartition);
    short offsetFetchErrorCode = result.error();

    if (offsetFetchErrorCode == ErrorMapping.NotCoordinatorForConsumerCode()) {
        log.error("Error encountered whilst fetching Kafka offset: NotCoordinatorForConsumerCode");
        return -1;
    } else if (offsetFetchErrorCode == ErrorMapping.OffsetsLoadInProgressCode()) {
        log.error("Error encountered whilst fetching Kafka offset: OffsetsLoadInProgressCode");
        return -1;
    } else {
        long retrievedOffset = result.offset();
        String retrievedMetadata = result.metadata();
        log.debug("Received offsets for topic " + topic + " & partition " + partition);
        log.debug("Offset: " + String.valueOf(retrievedOffset) + " Metadata: " + retrievedMetadata);

        // if broker has returned -1 without error, we've yet to commit.
        // start to read from 0
        if (retrievedOffset == -1) {
            log.info("No commits found against Kafka queue for topic "
                    + topic + " & partition " + partition + ". Setting read offset to 0");
            return 0;
        } else {
            return retrievedOffset;
        }
    }
}