Java PartitionOffsetRequestInfo類代碼示例

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
private static OffsetInfo getOffset(String topic, PartitionMetadata partition) {
  Broker broker = partition.leader();

  SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 10000, 1000000,
                                               "com.rekko.newrelic.storm.kafka");
  try {
    TopicAndPartition
        topicAndPartition =
        new TopicAndPartition(topic, partition.partitionId());
    PartitionOffsetRequestInfo rquest = new PartitionOffsetRequestInfo(-1, 1);
    Map<TopicAndPartition, PartitionOffsetRequestInfo>
        map =
        new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    map.put(topicAndPartition, rquest);
    OffsetRequest req = new OffsetRequest(map, (short) 0, "com.rekko.newrelic.storm.kafka");
    OffsetResponse resp = consumer.getOffsetsBefore(req);
    OffsetInfo offset = new OffsetInfo();
    offset.offset = resp.offsets(topic, partition.partitionId())[0];
    return offset;
  } finally {
    consumer.close();
  }
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String
        clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition,
            PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic,
                partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
/**
 * Defines where to start reading data from
 * Helpers Available:
 * kafka.api.OffsetRequest.EarliestTime() => finds the beginning of the data in the logs and starts streaming
 * from there
 * kafka.api.OffsetRequest.LatestTime()   => will only stream new messages
 *
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
/***
     * Get the offset of the last message in the specified topic and partition
     *
     * @param consumer   consumer object
     * @param topic      topic name
     * @param partition  partition id
     * @param whichTime  time
     * @param clientName client name
     * @return offset value
     * @throws Exception if leader is not found or there is an error fetching data offset from broker.
     */
    public static long getLastOffset(kafka.javaapi.consumer.SimpleConsumer consumer, String topic, int partition,
                                     long whichTime, String clientName) throws Exception {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
        OffsetResponse response = consumer.getOffsetsBefore(request);

        if (response.hasError()) {
            throw new Exception("Error fetching data Offset from the Broker. Reason: " + response.errorCode(topic, partition));
//            return 0;
        }

        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request =
      new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
/**
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return 0 if consumer is null at this time
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
  if (consumer == null) {
    return 0;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
@Override
public synchronized long getLatestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
    return -1;
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
@Override
public synchronized long getEarliestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.EarliestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }

    return -1;
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);

    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }

    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());

    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
public static long getLastOffset(SimpleConsumer consumer, String topic,
		int partition, long whichTime, String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
			partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
			whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
			requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
			clientName);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out
				.println("Error fetching data Offset Data the Broker. Reason: "
						+ response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) );
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
public long fetchResetOffset(String reset){
		long time = LatestTime();
		if (reset != null && reset.equals(SmallestTimeString()))
			time = EarliestTime();
		Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
		TopicAndPartition tp = new TopicAndPartition(topic, partition);
		PartitionOffsetRequestInfo info = new PartitionOffsetRequestInfo(time,1);
		requestInfo.put(tp, info);
		OffsetRequest request = new OffsetRequest(requestInfo,CurrentVersion(), clientId);
		OffsetResponse response = consumer.getOffsetsBefore(request);
		if (response.hasError()) {
			//ErrorMapping.exceptionFor(response.errorCode(topic, partition)).printStackTrace();
			throw new KafkaPartitionReaderException(response.errorCode(topic, partition));
		}
		long[] offsets = response.offsets(topic, partition);
		//TODO: confirm with xiaoju why we need this check?
//		if (offsets.length <= 0)
//			continue;
		return offsets[0];
	}
 

import kafka.api.PartitionOffsetRequestInfo; //導入依賴的package包/類
private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId)
{
    TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId);

    // The API implies that this will always return all of the offsets. So it seems a partition can not have
    // more than Integer.MAX_VALUE-1 segments.
    //
    // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value
    // should not be 0.
    PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE);
    OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
    OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest);

    if (offsetResponse.hasError()) {
        short errorCode = offsetResponse.errorCode(topicName, partitionId);
        log.warn("Offset response has error: %d", errorCode);
        throw new PrestoException(KAFKA_SPLIT_ERROR, "could not fetch data from Kafka, error code is '" + errorCode + "'");
    }

    return offsetResponse.offsets(topicName, partitionId);
}