Java OffsetRequest.EarliestTime方法代码示例

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
/**
 * Creates Kafka spout.
 *
 * @param topic Kafka topic
 * @return {@link KafkaSpout}
 */
protected org.apache.storm.kafka.KafkaSpout createKafkaSpout(String topic, String spoutId) {
    String zkRoot = String.format("/%s/%s", getTopologyName(), topic);
    ZkHosts hosts = new ZkHosts(config.getZookeeperHosts());

    SpoutConfig cfg = new SpoutConfig(hosts, topic, zkRoot, spoutId);
    cfg.startOffsetTime = OffsetRequest.EarliestTime();
    cfg.scheme = new SchemeAsMultiScheme(new StringScheme());
    cfg.bufferSizeBytes = 1024 * 1024 * 4;
    cfg.fetchSizeBytes = 1024 * 1024 * 4;

    return new org.apache.storm.kafka.KafkaSpout(cfg);
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
/**
 * For a set of partitions, if a partition is set with the special offsets {@link OffsetRequest#EarliestTime()}
 * or {@link OffsetRequest#LatestTime()}, replace them with actual offsets requested via a Kafka consumer.
 *
 * @param consumer The consumer connected to lead broker
 * @param partitions The list of partitions we need offsets for
 */
private static void requestAndSetEarliestOrLatestOffsetsFromKafka(
		SimpleConsumer consumer,
		List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws Exception {
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
	for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) {
		if (part.getOffset() == OffsetRequest.EarliestTime() || part.getOffset() == OffsetRequest.LatestTime()) {
			requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(part.getOffset(), 1));
		}
	}

	requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo);
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
/**
 * Retrieve the behaviour of "auto.offset.reset" from the config properties.
 * A partition needs to fallback to "auto.offset.reset" as default offset when
 * we can't find offsets in ZK to start from in {@link StartupMode#GROUP_OFFSETS} startup mode.
 *
 * @param config kafka consumer properties
 * @return either OffsetRequest.LatestTime() or OffsetRequest.EarliestTime()
 */
private static long getInvalidOffsetBehavior(Properties config) {
	final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
	if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
		return OffsetRequest.LatestTime();
	} else {
		return OffsetRequest.EarliestTime();
	}
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
private static long getInvalidOffsetBehavior(Properties config) {
	final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
	if (val.equals("none")) {
		throw new IllegalArgumentException("Cannot use '" + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG
				+ "' value 'none'. Possible values: 'latest', 'largest', or 'earliest'.");
	}
	else if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
		return OffsetRequest.LatestTime();
	} else {
		return OffsetRequest.EarliestTime();
	}
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
@Test
public void getOffsetFromConfigAndDontForceFromStart() {
    config.forceFromStart = false;
    config.startOffsetTime = OffsetRequest.EarliestTime();
    createTopicAndSendMessage();
    long latestOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime());
    long offsetFromConfig = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, config);
    assertThat(latestOffset, is(equalTo(offsetFromConfig)));
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
@Test
public void getOffsetFromConfigAndFroceFromStart() {
    config.forceFromStart = true;
    config.startOffsetTime = OffsetRequest.EarliestTime();
    createTopicAndSendMessage();
    long earliestOffset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.EarliestTime());
    long offsetFromConfig = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, config);
    assertThat(earliestOffset, is(equalTo(offsetFromConfig)));
}