Java OffsetAndMetadata.offset方法代码示例

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
/**
 * sets the cursor to the location dictated by the first poll strategy and returns the fetch offset
 */
private long doSeek(TopicPartition tp, OffsetAndMetadata committedOffset) {
    long fetchOffset;
    if (committedOffset != null) {             // offset was committed for this TopicPartition
        if (firstPollOffsetStrategy.equals(EARLIEST)) {
            kafkaConsumer.seekToBeginning(Collections.singleton(tp));
            fetchOffset = kafkaConsumer.position(tp);
        } else if (firstPollOffsetStrategy.equals(LATEST)) {
            kafkaConsumer.seekToEnd(Collections.singleton(tp));
            fetchOffset = kafkaConsumer.position(tp);
        } else {
            // By default polling starts at the last committed offset. +1 to point fetch to the first uncommitted offset.
            fetchOffset = committedOffset.offset() + 1;
            kafkaConsumer.seek(tp, fetchOffset);
        }
    } else {    // no commits have ever been done, so start at the beginning or end depending on the strategy
        if (firstPollOffsetStrategy.equals(EARLIEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_EARLIEST)) {
            kafkaConsumer.seekToBeginning(Collections.singleton(tp));
        } else if (firstPollOffsetStrategy.equals(LATEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_LATEST)) {
            kafkaConsumer.seekToEnd(Collections.singleton(tp));
        }
        fetchOffset = kafkaConsumer.position(tp);
    }
    return fetchOffset;
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
private void checkIfRefreshCommitRequired() {
    // Here's the issue:
    // The retention of __consumer_offsets is less than most topics itself, so we need to re-commit regularly to keep the
    // last committed offset per consumer group. This is especially an issue in cases were we have bursty / little traffic.

    Map<TopicPartition, OffsetAndMetadata> commitOffsets = new HashMap<>();
    long now = System.currentTimeMillis();

    if (nextCommitRefreshRequiredTimestamp < now) {
        nextCommitRefreshRequiredTimestamp = now + COMMIT_REFRESH_INTERVAL_MILLIS;

        for (PartitionProcessor processor : partitions.allProcessors()) {
            TopicPartition assignedPartition = processor.getAssignedPartition();
            long lastCommittedOffset = processor.getLastCommittedOffset();

            // We haven't committed from this partiton yet
            if (lastCommittedOffset < 0) {
                OffsetAndMetadata offset = kafka.committed(assignedPartition);
                if (offset == null) {
                    // there was no commit on this partition at all
                    continue;
                }
                lastCommittedOffset = offset.offset();
                processor.forceSetLastCommittedOffset(lastCommittedOffset);
            }


            commitOffsets.put(assignedPartition, new OffsetAndMetadata(lastCommittedOffset));
        }


        kafka.commitSync(commitOffsets);

        logger.info("Refreshing last committed offset {}", commitOffsets);
    }


}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
@Test
public void subscriberLosesPartitionAssignment() {
    KafkaSubscriber<String> subscriber = new KafkaSubscriber<>(new MessageCallback(),
            "topic", "groupId", false,
            KafkaSubscriber.OffsetReset.Earliest, 1, 1, 1,
            5000, 5000);
    KafkaTopicInfo message1 = new KafkaTopicInfo("topic", 0, 1, null);
    KafkaTopicInfo message2 = new KafkaTopicInfo("topic", 0, 2, null);
    KafkaTopicInfo message3 = new KafkaTopicInfo("topic", 1, 1, null);
    KafkaTopicInfo message4 = new KafkaTopicInfo("topic", 1, 2, null);
    subscriber.consume(message1);
    subscriber.consume(message2);
    subscriber.consume(message3);
    subscriber.consume(message4);
    KafkaConsumer realConsumer = mock(KafkaConsumer.class);
    class ArgMatcher implements ArgumentMatcher<Map<TopicPartition, OffsetAndMetadata>> {
        @Override
        public boolean matches(Map<TopicPartition, OffsetAndMetadata> arg) {
            OffsetAndMetadata oam = arg.values().iterator().next();
            return oam.offset() == 3;
        }
    }
    doThrow(new CommitFailedException()).when(realConsumer).commitSync(argThat(new ArgMatcher()));
    subscriber.realConsumer = realConsumer;
    subscriber.offsetCommitter = new OffsetCommitter(realConsumer, Clock.systemUTC());
    subscriber.consumeMessages();
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
/**
 * 按上次记录重置offsets
 */
private void resetCorrectOffsets() {
    consumer.pause(consumer.assignment());
    Map<String, List<PartitionInfo>> topicInfos = consumer.listTopics();
    Set<String> topics = topicInfos.keySet();

    List<String> expectTopics = new ArrayList<>(topicHandlers.keySet());

    List<PartitionInfo> patitions = null;
    for (String topic : topics) {
        if (!expectTopics.contains(topic))
            continue;

        patitions = topicInfos.get(topic);
        for (PartitionInfo partition : patitions) {
            try {
                //期望的偏移
                long expectOffsets = consumerContext.getLatestProcessedOffsets(topic,
                    partition.partition());
                //
                TopicPartition topicPartition = new TopicPartition(topic,
                    partition.partition());
                OffsetAndMetadata metadata = consumer
                    .committed(new TopicPartition(partition.topic(), partition.partition()));
                if (expectOffsets >= 0) {
                    if (expectOffsets < metadata.offset()) {
                        consumer.seek(topicPartition, expectOffsets);
                        logger.info("seek Topic[{}] partition[{}] from {} to {}", topic,
                            partition.partition(), metadata.offset(), expectOffsets);
                    }
                }
            } catch (Exception e) {
                logger.warn("try seek topic[" + topic + "] partition[" + partition.partition()
                            + "] offsets error");
            }
        }
    }
    consumer.resume(consumer.assignment());
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
private TxnOffsetCommitHandler txnOffsetCommitHandler(TransactionalRequestResult result,
                                                      Map<TopicPartition, OffsetAndMetadata> offsets,
                                                      String consumerGroupId) {
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        CommittedOffset committedOffset = new CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata());
        pendingTxnOffsetCommits.put(entry.getKey(), committedOffset);
    }
    TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(transactionalId, consumerGroupId,
            producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, pendingTxnOffsetCommits);
    return new TxnOffsetCommitHandler(result, builder);
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
/**
 * Don't support multiple subscribe on topic partition.
 */
public Long getLastCommittedOffset(TopicPartition topicPartition) {
    List<ProxyMockConsumer> proxyMockConsumers = existingOpenedConsumers(topicPartition.topic());
    OffsetAndMetadata map = proxyMockConsumers.get(0).committed(topicPartition);
    return map == null ? null : map.offset();
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
private void commitOffsets(long now, boolean closing) {
    if (currentOffsets.isEmpty())
        return;

    committing = true;
    commitSeqno += 1;
    commitStarted = now;

    final Map<TopicPartition, OffsetAndMetadata> taskProvidedOffsets;
    try {
        taskProvidedOffsets = task.preCommit(new HashMap<>(currentOffsets));
    } catch (Throwable t) {
        if (closing) {
            log.warn("{} Offset commit failed during close");
            onCommitCompleted(t, commitSeqno);
        } else {
            log.error("{} Offset commit failed, rewinding to last committed offsets", this, t);
            for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : lastCommittedOffsets.entrySet()) {
                log.debug("{} Rewinding topic partition {} to offset {}", id, entry.getKey(), entry.getValue().offset());
                consumer.seek(entry.getKey(), entry.getValue().offset());
            }
            currentOffsets = new HashMap<>(lastCommittedOffsets);
            onCommitCompleted(t, commitSeqno);
        }
        return;
    } finally {
        // Close the task if needed before committing the offsets.
        if (closing)
            task.close(currentOffsets.keySet());
    }

    if (taskProvidedOffsets.isEmpty()) {
        log.debug("{} Skipping offset commit, task opted-out", this);
        onCommitCompleted(null, commitSeqno);
        return;
    }

    final Map<TopicPartition, OffsetAndMetadata> commitableOffsets = new HashMap<>(lastCommittedOffsets);
    for (Map.Entry<TopicPartition, OffsetAndMetadata> taskProvidedOffsetEntry : taskProvidedOffsets.entrySet()) {
        final TopicPartition partition = taskProvidedOffsetEntry.getKey();
        final OffsetAndMetadata taskProvidedOffset = taskProvidedOffsetEntry.getValue();
        if (commitableOffsets.containsKey(partition)) {
            if (taskProvidedOffset.offset() <= currentOffsets.get(partition).offset()) {
                commitableOffsets.put(partition, taskProvidedOffset);
            } else {
                log.warn("Ignoring invalid task provided offset {}/{} -- not yet consumed", partition, taskProvidedOffset);
            }
        } else {
            log.warn("Ignoring invalid task provided offset {}/{} -- partition not assigned", partition, taskProvidedOffset);
        }
    }

    if (commitableOffsets.equals(lastCommittedOffsets)) {
        log.debug("{} Skipping offset commit, no change since last commit", this);
        onCommitCompleted(null, commitSeqno);
        return;
    }

    log.trace("{} Offsets to commit: {}", this, commitableOffsets);
    doCommit(commitableOffsets, closing, commitSeqno);
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
/**
 * Commit offsets for the specified list of topics and partitions. This is a non-blocking call
 * which returns a request future that can be polled in the case of a synchronous commit or ignored in the
 * asynchronous case.
 *
 * @param offsets The list of offsets per partition that should be committed.
 * @return A request future whose value indicates whether the commit was successful or not
 */
private RequestFuture<Void> sendOffsetCommitRequest(final Map<TopicPartition, OffsetAndMetadata> offsets) {
    if (offsets.isEmpty())
        return RequestFuture.voidSuccess();

    Node coordinator = coordinator();
    if (coordinator == null)
        return RequestFuture.coordinatorNotAvailable();

    // create the offset commit request
    Map<TopicPartition, OffsetCommitRequest.PartitionData> offsetData = new HashMap<>(offsets.size());
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        if (offsetAndMetadata.offset() < 0) {
            return RequestFuture.failure(new IllegalArgumentException("Invalid offset: " + offsetAndMetadata.offset()));
        }
        offsetData.put(entry.getKey(), new OffsetCommitRequest.PartitionData(
                offsetAndMetadata.offset(), offsetAndMetadata.metadata()));
    }

    final Generation generation;
    if (subscriptions.partitionsAutoAssigned())
        generation = generation();
    else
        generation = Generation.NO_GENERATION;

    // if the generation is null, we are not part of an active group (and we expect to be).
    // the only thing we can do is fail the commit and let the user rejoin the group in poll()
    if (generation == null)
        return RequestFuture.failure(new CommitFailedException());

    OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(this.groupId, offsetData).
            setGenerationId(generation.generationId).
            setMemberId(generation.memberId).
            setRetentionTime(OffsetCommitRequest.DEFAULT_RETENTION_TIME);

    log.trace("Sending OffsetCommit request with {} to coordinator {} for group {}", offsets, coordinator, groupId);
    //compose会调用 RequestFutureAdapter 的onSuccess
    //OffsetCommitResponseHandler extends CoordinatorResponseHandle
    //调用CoordinatorResponseHandle的onSuccess方法
    //JoinGroupResponseHandler调用其handle方法并调用onJoinLeader
    //在onJoinLeader中调用performAssignment方法
    //performAssignment 在本类中实现
    return client.send(coordinator, builder)
            .compose(new OffsetCommitResponseHandler(offsets));
}
 

import org.apache.kafka.clients.consumer.OffsetAndMetadata; //导入方法依赖的package包/类
@Override
public void handle(OffsetCommitResponse commitResponse, RequestFuture<Void> future) {
    sensors.commitLatency.record(response.requestLatencyMs());
    Set<String> unauthorizedTopics = new HashSet<>();

    for (Map.Entry<TopicPartition, Errors> entry : commitResponse.responseData().entrySet()) {
        TopicPartition tp = entry.getKey();
        OffsetAndMetadata offsetAndMetadata = this.offsets.get(tp);
        long offset = offsetAndMetadata.offset();

        Errors error = entry.getValue();
        if (error == Errors.NONE) {
            log.debug("Group {} committed offset {} for partition {}", groupId, offset, tp);
            if (subscriptions.isAssigned(tp))
                // update the local cache only if the partition is still assigned
                subscriptions.committed(tp, offsetAndMetadata);
        } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
            log.error("Not authorized to commit offsets for group {}", groupId);
            future.raise(new GroupAuthorizationException(groupId));
            return;
        } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
            unauthorizedTopics.add(tp.topic());
        } else if (error == Errors.OFFSET_METADATA_TOO_LARGE
                || error == Errors.INVALID_COMMIT_OFFSET_SIZE) {
            // raise the error to the user
            log.debug("Offset commit for group {} failed on partition {}: {}", groupId, tp, error.message());
            future.raise(error);
            return;
        } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS) {
            // just retry
            log.debug("Offset commit for group {} failed: {}", groupId, error.message());
            future.raise(error);
            return;
        } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                || error == Errors.NOT_COORDINATOR
                || error == Errors.REQUEST_TIMED_OUT) {
            log.debug("Offset commit for group {} failed: {}", groupId, error.message());
            coordinatorDead();
            future.raise(error);
            return;
        } else if (error == Errors.UNKNOWN_MEMBER_ID
                || error == Errors.ILLEGAL_GENERATION
                || error == Errors.REBALANCE_IN_PROGRESS) {
            // need to re-join group
            log.debug("Offset commit for group {} failed: {}", groupId, error.message());
            resetGeneration();
            future.raise(new CommitFailedException());
            return;
        } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
            log.debug("Offset commit for group {} failed on partition {}: {}", groupId, tp, error.message());
            future.raise(new KafkaException("Partition " + tp + " may not exist or user may not have Describe access to topic"));
            return;
        } else {
            log.error("Group {} failed to commit partition {} at offset {}: {}", groupId, tp, offset, error.message());
            future.raise(new KafkaException("Unexpected error in commit: " + error.message()));
            return;
        }
    }

    if (!unauthorizedTopics.isEmpty()) {
        log.error("Not authorized to commit to topics {} for group {}", unauthorizedTopics, groupId);
        future.raise(new TopicAuthorizationException(unauthorizedTopics));
    } else {
        future.complete(null);
    }
}