Java OffsetRequest.LatestTime方法代码示例

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
private List<long[]> generateTsOffsetsNoPartitions(final long scanRange,
                                                   final int partitionNum)
{
    // we want to increase the number of splits so that it can achieve maximum parallelism
    // the idle number would be splits == cores
    // TODO make this configerable
    final int numHosts = 40;
    final int numCorePerHosts = 32;
    final int splitsWanted = numHosts * numCorePerHosts;
    final long start = System.currentTimeMillis() - scanRange;
    long secondsPerSplit = scanRange / (splitsWanted / partitionNum);

    List<long[]> offsetList = new ArrayList<long[]>();
    for (int i = 0; i < splitsWanted / partitionNum; ++i) {
        long[] offsets = new long[2];
        offsets[0] = start + secondsPerSplit * i;
        offsets[1] = start + secondsPerSplit * (i + 1);
        offsetList.add(offsets);
    }
    offsetList.get(offsetList.size() - 1)[1] = OffsetRequest.LatestTime();
    return offsetList;
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
/**
 * For a set of partitions, if a partition is set with the special offsets {@link OffsetRequest#EarliestTime()}
 * or {@link OffsetRequest#LatestTime()}, replace them with actual offsets requested via a Kafka consumer.
 *
 * @param consumer The consumer connected to lead broker
 * @param partitions The list of partitions we need offsets for
 */
private static void requestAndSetEarliestOrLatestOffsetsFromKafka(
		SimpleConsumer consumer,
		List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws Exception {
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
	for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) {
		if (part.getOffset() == OffsetRequest.EarliestTime() || part.getOffset() == OffsetRequest.LatestTime()) {
			requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(part.getOffset(), 1));
		}
	}

	requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo);
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
/**
 * Retrieve the behaviour of "auto.offset.reset" from the config properties.
 * A partition needs to fallback to "auto.offset.reset" as default offset when
 * we can't find offsets in ZK to start from in {@link StartupMode#GROUP_OFFSETS} startup mode.
 *
 * @param config kafka consumer properties
 * @return either OffsetRequest.LatestTime() or OffsetRequest.EarliestTime()
 */
private static long getInvalidOffsetBehavior(Properties config) {
	final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
	if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
		return OffsetRequest.LatestTime();
	} else {
		return OffsetRequest.EarliestTime();
	}
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
private static long getInvalidOffsetBehavior(Properties config) {
	final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest");
	if (val.equals("none")) {
		throw new IllegalArgumentException("Cannot use '" + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG
				+ "' value 'none'. Possible values: 'latest', 'largest', or 'earliest'.");
	}
	else if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9
		return OffsetRequest.LatestTime();
	} else {
		return OffsetRequest.EarliestTime();
	}
}
 

import kafka.api.OffsetRequest; //导入方法依赖的package包/类
private long getTopicLogSize(String topic, int pid) {
  Option<Object> o = ZkUtils.getLeaderForPartition(zkClient, topic, pid);
  if (o.isEmpty() || o.get() == null) {
    log.error("No broker for partition %s - %s", topic, pid);
    return 0;
  }
  Integer leaderId = Int.unbox(o.get());
  SimpleConsumer consumer = consumerMap.get(leaderId);
  if (consumer == null) {
    consumer = createSimpleConsumer(leaderId);
  }
  // createSimpleConsumer may fail.
  if (consumer == null) {
    return 0;
  }
  consumerMap.put(leaderId, consumer);
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, pid);
  PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1);
  OffsetRequest request = new OffsetRequest(
    new Map1<TopicAndPartition, PartitionOffsetRequestInfo>(topicAndPartition, requestInfo),
    0,
    Request.OrdinaryConsumerId()
  );
  OffsetResponse response = consumer.getOffsetsBefore(request);
  PartitionOffsetsResponse offsetsResponse = response.partitionErrorAndOffsets().get(topicAndPartition).get();
  return scala.Long.unbox(offsetsResponse.offsets().head());
}