Java OffsetCommitCallback类代码示例

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
private void doAutoCommitOffsetsAsync() {
    Map<TopicPartition, OffsetAndMetadata> allConsumedOffsets = subscriptions.allConsumed();
    log.debug("Sending asynchronous auto-commit of offsets {} for group {}", allConsumedOffsets, groupId);
    //rebalance
    commitOffsetsAsync(allConsumedOffsets, new OffsetCommitCallback() {
        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            if (exception != null) {
                log.warn("Auto-commit of offsets {} failed for group {}: {}", offsets, groupId,
                        exception.getMessage());
                if (exception instanceof RetriableException)
                    nextAutoCommitDeadline = Math.min(time.milliseconds() + retryBackoffMs, nextAutoCommitDeadline);
            } else {
                log.debug("Completed auto-commit of offsets {} for group {}", offsets, groupId);
            }
        }
    });
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
public void commitOffset(long offset) {
    OffsetAndMetadata offsetMeta = new OffsetAndMetadata(offset + 1, "");

    Map<TopicPartition, OffsetAndMetadata> offsetMap = new HashMap<>();
    offsetMap.put(statTopicPartition, offsetMeta);

    OffsetCommitCallback callback = new OffsetCommitCallback() {
        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> map, Exception e) {
            if (e != null) {
                LOG.warn(String.format("CommitAsync failed!!!! offset %d, Topic %s", offset, statTopic));
            }
            else {
                ; //do nothing when OK;
                //logger.info(String.format("OK. offset %d, Topic %s", record.offset(), record.topic()));
            }
        }
    };
    consumer.commitAsync(offsetMap, callback);
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
@Override
public void commitAsync(final OffsetCommitCallback callback) {
    Retries.tryMe(new IgniteInClosure<RetryRunnableAsyncOnCallback>() {
        @Override
        public void apply(final RetryRunnableAsyncOnCallback retryRunnableAsyncOnCallback) {
            inner.commitAsync(new OffsetCommitCallback() {
                @Override
                public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
                    callback.onComplete(offsets, exception);
                    if (exception != null) {
                        retryRunnableAsyncOnCallback.retry(exception);
                    }
                }
            });
        }
    }, strategy());
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
private void doCommitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
    this.subscriptions.needRefreshCommits();
    //rebalance
    RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
    final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback;
    future.addListener(new RequestFutureListener<Void>() {
        @Override
        public void onSuccess(Void value) {
            if (interceptors != null)
                interceptors.onCommit(offsets);

            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, null));
        }

        @Override
        public void onFailure(RuntimeException e) {
            Exception commitException = e;

            if (e instanceof RetriableException)
                commitException = RetriableCommitFailedException.withUnderlyingMessage(e.getMessage());

            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, commitException));
        }
    });
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
static void commitOffsets(Consumer<?, ?> consumer, Map<TopicPartition, OffsetAndMetadata> offsetsToCommit,
                          boolean async) {

    if (offsetsToCommit == null || offsetsToCommit.isEmpty()) {
        return;
    }

    OffsetCommitCallback callback = (offsets, exception) -> {
        if (exception != null) {
            LOG.warn("Unable to commit offsets for {} TopicPartition(s) {}: {}",
                    offsets.size(),
                    offsetsAsString(offsets),
                    exception.getMessage(),
                    exception);
        } else {
            LOG.debug("Successfully committed offset(s) for {} TopicPartition(s): {}",
                    offsets.size(), offsetsAsString(offsets));
        }
    };

    if (async) {
        consumer.commitAsync(offsetsToCommit, callback);
    } else {
        consumer.commitSync(offsetsToCommit);
    }
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
public void run(final long now) {
    if (coordinatorUnknown()) {
        log.debug("Cannot auto-commit offsets for group {} since the coordinator is unknown", groupId);
        reschedule(now + retryBackoffMs);
        return;
    }

    if (needRejoin()) {
        // skip the commit when we're rejoining since we'll commit offsets synchronously
        // before the revocation callback is invoked
        reschedule(now + interval);
        return;
    }

    commitOffsetsAsync(subscriptions.allConsumed(), new OffsetCommitCallback() {
        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            if (exception == null) {
                reschedule(now + interval);
            } else {
                log.warn("Auto offset commit failed for group {}: {}", groupId, exception.getMessage());
                reschedule(now + interval);
            }
        }
    });
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
@Override
public void commit(Map<TopicPartition, OffsetAndMetadata> offsets, Handler<AsyncResult<Map<TopicPartition, OffsetAndMetadata>>> completionHandler) {
  this.submitTask((consumer, future) -> {
    OffsetCommitCallback callback = (result, exception) -> {
      if (future != null) {
        if (exception != null) {
          future.fail(exception);
        } else {
          future.complete(result);
        }
      }
    };
    if (offsets == null) {
      consumer.commitAsync(callback);
    } else {
      consumer.commitAsync(offsets, callback);
    }
  }, completionHandler);
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
private Capture<OffsetCommitCallback> expectOffsetCommit(final long expectedMessages,
                                                         final RuntimeException error,
                                                         final Exception consumerCommitError,
                                                         final long consumerCommitDelayMs,
                                                         final boolean invokeCallback)
        throws Exception {
    final long finalOffset = FIRST_OFFSET + expectedMessages;

    // All assigned partitions will have offsets committed, but we've only processed messages/updated offsets for one
    final Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();
    offsetsToCommit.put(TOPIC_PARTITION, new OffsetAndMetadata(finalOffset));
    offsetsToCommit.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    offsetsToCommit.put(TOPIC_PARTITION3, new OffsetAndMetadata(FIRST_OFFSET));
    sinkTask.preCommit(offsetsToCommit);
    IExpectationSetters<Object> expectation = PowerMock.expectLastCall();
    if (error != null) {
        expectation.andThrow(error).once();
        return null;
    } else {
        expectation.andReturn(offsetsToCommit);
    }

    final Capture<OffsetCommitCallback> capturedCallback = EasyMock.newCapture();
    consumer.commitAsync(EasyMock.eq(offsetsToCommit),
            EasyMock.capture(capturedCallback));
    PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() {
        @Override
        public Object answer() throws Throwable {
            time.sleep(consumerCommitDelayMs);
            if (invokeCallback)
                capturedCallback.getValue().onComplete(offsetsToCommit, consumerCommitError);
            return null;
        }
    });
    return capturedCallback;
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
public void commitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
    invokeCompletedOffsetCommitCallbacks();
    // 没有找到可用的Node
    if (!coordinatorUnknown()) {
        //rebalance
        doCommitOffsetsAsync(offsets, callback);
    } else {
        // we don't know the current coordinator, so try to find it and then send the commit
        // or fail (we don't want recursive retries which can cause offset commits to arrive
        // out of order). Note that there may be multiple offset commits chained to the same
        // coordinator lookup request. This is fine because the listeners will be invoked in
        // the same order that they were added. Note also that AbstractCoordinator prevents
        // multiple concurrent coordinator lookup requests.
        pendingAsyncCommits.incrementAndGet();
        lookupCoordinator().addListener(new RequestFutureListener<Void>() {
            @Override
            public void onSuccess(Void value) {
                pendingAsyncCommits.decrementAndGet();
                doCommitOffsetsAsync(offsets, callback);
            }

            @Override
            public void onFailure(RuntimeException e) {
                pendingAsyncCommits.decrementAndGet();
                completedOffsetCommits.add(new OffsetCommitCompletion(callback, offsets,
                        RetriableCommitFailedException.withUnderlyingMessage(e.getMessage())));
            }
        });
    }

    // ensure the commit has a chance to be transmitted (without blocking on its completion).
    // Note that commits are treated as heartbeats by the coordinator, so there is no need to
    // explicitly allow heartbeats through delayed task execution.
    client.pollNoWakeup();
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
private OffsetCommitCallback callback(final AtomicBoolean success) {
    return new OffsetCommitCallback() {
        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            if (exception == null)
                success.set(true);
        }
    };
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
/**
 * Use the supplied function to asynchronously consume messages from the cluster.
 *
 * @param groupId              the name of the group; may not be null
 * @param clientId             the name of the client; may not be null
 * @param autoOffsetReset      how to pick a starting offset when there is no initial offset in ZooKeeper or if an offset is
 *                             out of range; may be null for the default to be used
 * @param keyDeserializer      the deserializer for the keys; may not be null
 * @param valueDeserializer    the deserializer for the values; may not be null
 * @param continuation         the function that determines if the consumer should continue; may not be null
 * @param offsetCommitCallback the callback that should be used after committing offsets; may be null if offsets are
 *                             not to be committed
 * @param completion           the function to call when the consumer terminates; may be null
 * @param topics               the set of topics to consume; may not be null or empty
 * @param consumerFunction     the function to consume the messages; may not be null
 */
public <K, V> void consume(String groupId, String clientId, OffsetResetStrategy autoOffsetReset,
                           Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer,
                           BooleanSupplier continuation, OffsetCommitCallback offsetCommitCallback, Runnable completion,
                           Collection<String> topics,
                           java.util.function.Consumer<ConsumerRecord<K, V>> consumerFunction) {
    Properties props = getConsumerProperties(groupId, clientId, autoOffsetReset);
    Thread t = new Thread(() -> {
        LOGGER.info("Starting consumer {} to read messages", clientId);
        try (KafkaConsumer<K, V> consumer = new KafkaConsumer<>(props, keyDeserializer, valueDeserializer)) {
            consumer.subscribe(new ArrayList<>(topics));
            while (continuation.getAsBoolean()) {
                consumer.poll(10).forEach(record -> {
                    LOGGER.info("Consumer {}: consuming message {}", clientId, record);
                    consumerFunction.accept(record);
                    if (offsetCommitCallback != null) {
                        consumer.commitAsync(offsetCommitCallback);
                    }
                });
            }
        } finally {
            if (completion != null) completion.run();
            LOGGER.debug("Stopping consumer {}", clientId);
        }
    });
    t.setName(clientId + "-thread");
    t.start();
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
public void consumeIntegers(BooleanSupplier continuation, Runnable completion, Collection<String> topics, Consumer<ConsumerRecord<String, Integer>> consumerFunction) {
    Deserializer<String> keyDes = new StringDeserializer();
    Deserializer<Integer> valDes = new IntegerDeserializer();
    String randomId = UUID.randomUUID().toString();
    OffsetCommitCallback offsetCommitCallback = null;
    this.consume(randomId, randomId, OffsetResetStrategy.EARLIEST, keyDes, valDes, continuation, (OffsetCommitCallback) offsetCommitCallback, completion, topics, consumerFunction);
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
private void commitOffsets(Map<TopicPartition, OffsetAndMetadata> offsets,
                           boolean ignoreConsumerHighWatermark,
                           OffsetCommitCallback callback,
                           boolean sync) {
  Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = getOffsetsToCommit(offsets, ignoreConsumerHighWatermark);
  if (sync) {
    LOG.trace("Committing offsets synchronously: {}", offsetsToCommit);
    _kafkaConsumer.commitSync(offsetsToCommit);
  } else {
    LOG.trace("Committing offsets asynchronously: {}", offsetsToCommit);
    _offsetCommitCallback.setUserCallback(callback);
    _kafkaConsumer.commitAsync(offsetsToCommit, _offsetCommitCallback);
  }
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
@Test
public void testOffsetCommitCallback() {
  String topic = "testOffsetCommitCallback";
  produceSyntheticMessages(topic);
  Properties props = new Properties();
  // All the consumers should have the same group id.
  props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "testOffsetCommitCallback");
  // Make sure we start to consume from the beginning.
  props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  // Only fetch one record at a time.
  props.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "1");
  try (LiKafkaConsumer<String, String> consumer = createConsumer(props)) {
    TopicPartition tp = new TopicPartition(topic, 0);
    consumer.assign(Collections.singleton(tp));
    consumer.poll(5000); // M2
    final AtomicBoolean offsetCommitted = new AtomicBoolean(false);
    consumer.commitAsync(new OffsetCommitCallback() {
      @Override
      public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception e) {
        assertEquals(topicPartitionOffsetAndMetadataMap.get(tp), new OffsetAndMetadata(3, ""), "The committed user offset should be 3");
        offsetCommitted.set(true);
      }
    });
    while (!offsetCommitted.get()) {
      consumer.poll(10);
    }
    assertEquals(consumer.committed(tp), new OffsetAndMetadata(3, ""), "The committed user offset should be 3");
    assertEquals(consumer.committedSafeOffset(tp).longValue(), 0, "The committed actual offset should be 0");
  }
}
 

import org.apache.kafka.clients.consumer.OffsetCommitCallback; //导入依赖的package包/类
public void commitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
    this.subscriptions.needRefreshCommits();
    RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
    final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback;
    future.addListener(new RequestFutureListener<Void>() {
        @Override
        public void onSuccess(Void value) {
            if (interceptors != null)
                interceptors.onCommit(offsets);
            cb.onComplete(offsets, null);
        }

        @Override
        public void onFailure(RuntimeException e) {
            if (e instanceof RetriableException) {
                cb.onComplete(offsets, new RetriableCommitFailedException("Commit offsets failed with retriable exception. You should retry committing offsets.", e));
            } else {
                cb.onComplete(offsets, e);
            }
        }
    });

    // ensure the commit has a chance to be transmitted (without blocking on its completion).
    // Note that commits are treated as heartbeats by the coordinator, so there is no need to
    // explicitly allow heartbeats through delayed task execution.
    client.pollNoWakeup();
}