Java TopicAndPartition类代码示例

import kafka.common.TopicAndPartition; //导入依赖的package包/类
public long readOffset(String topic, int partition, String groupId) {
    BlockingChannel channel = new BlockingChannel(host, port,
            BlockingChannel.UseDefaultBufferSize(),
            BlockingChannel.UseDefaultBufferSize(),
            (int) Duration.ofSeconds(10).toMillis());
    try {
        channel.connect();
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(
                groupId, singletonList(topicAndPartition), (short) 1, RANDOM.nextInt(), "test-checker");
        channel.send(offsetFetchRequest.underlying());
        OffsetFetchResponse fetchResponse = OffsetFetchResponse.readFrom(channel.receive().payload());
        OffsetMetadataAndError result = fetchResponse.offsets().get(topicAndPartition);
        return result.offset();
    } finally {
        channel.disconnect();
    }
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private void updateLeaderMap() {
  for (String broker : brokerList) {
    try {
      SimpleConsumer consumer = getSimpleConsumer(broker);
      TopicMetadataRequest req = new TopicMetadataRequest(auditTopics);
      kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
      List<TopicMetadata> metaData = resp.topicsMetadata();

      for (TopicMetadata tmd : metaData) {
        for (PartitionMetadata pmd : tmd.partitionsMetadata()) {
          TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId());
          partitionLeader.put(topicAndPartition, getHostPort(pmd.leader()));
        }
      }
    } catch (Exception e) {
      logger.warn("Got exception to get metadata from broker=" + broker, e);
    }
  }
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private static OffsetInfo getOffset(String topic, PartitionMetadata partition) {
  Broker broker = partition.leader();

  SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 10000, 1000000,
                                               "com.rekko.newrelic.storm.kafka");
  try {
    TopicAndPartition
        topicAndPartition =
        new TopicAndPartition(topic, partition.partitionId());
    PartitionOffsetRequestInfo rquest = new PartitionOffsetRequestInfo(-1, 1);
    Map<TopicAndPartition, PartitionOffsetRequestInfo>
        map =
        new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    map.put(topicAndPartition, rquest);
    OffsetRequest req = new OffsetRequest(map, (short) 0, "com.rekko.newrelic.storm.kafka");
    OffsetResponse resp = consumer.getOffsetsBefore(req);
    OffsetInfo offset = new OffsetInfo();
    offset.offset = resp.offsets(topic, partition.partitionId())[0];
    return offset;
  } finally {
    consumer.close();
  }
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String
        clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition,
            PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic,
                partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private boolean commitOffset(SimpleConsumer consumer, long offset, String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, OffsetAndMetadata> requestInfo = new HashMap<TopicAndPartition, OffsetAndMetadata>();
	OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(
			new OffsetMetadata(offset, OffsetMetadata.NoMetadata()), offset, -1L);
	requestInfo.put(topicAndPartition, offsetAndMetadata);
	OffsetCommitRequest commitRequest = new OffsetCommitRequest(groupid, requestInfo, correlationId, clientName,
			OffsetRequest.CurrentVersion());
	OffsetCommitResponse response = null;
	while (true) {
		try {
			logger.debug("partition {} commit offest", partition);
			response = consumer.commitOffsets(commitRequest);
			if (response != null)
				break;
		} catch (Exception e) {
			logger.error("some error occur when fetch messages", e);
			try {
				Thread.sleep(EXCEPTION_SLEEP_TIME);
			} catch (InterruptedException e1) {
				e1.printStackTrace();
			}
		}
	}
	return response.hasError();
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private scala.collection.Map<TopicAndPartition, Seq<Object>> getAssignmentPlan(
    Map<TopicPartition, Integer[]> replicasMap) {
  scala.collection.mutable.HashMap<TopicAndPartition, Seq<Object>> result =
      new scala.collection.mutable.HashMap<>();

  for (Map.Entry<TopicPartition, Integer[]> entry : replicasMap.entrySet()) {
    TopicPartition tp = entry.getKey();
    TopicAndPartition tap = new TopicAndPartition(tp.topic(), tp.partition());
    List<Object> objs = Arrays.asList(entry.getValue()).stream()
        .map(val -> (Object) val).collect(Collectors.toList());
    Seq<Object> replicas = JavaConverters.asScalaBuffer(objs).seq();
    result.put(tap, replicas);
  }

  assert replicasMap.size() == result.size();
  LOG.debug("replicaMap.size = {}, result.size = {}", replicasMap.size(), result.size());
  return result;
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private static Long getOffset(OffsetResponse response, TopicAndPartition topicPartition) {
  String topic = topicPartition.topic();
  int partition = topicPartition.partition();
  long[] offsets = response.offsets(topic, partition);
  if (offsets.length > 0) {
    return offsets[0];
  }
  short errorCode = response.errorCode(topic, partition);
  if (errorCode == ErrorMapping.UnknownTopicOrPartitionCode()) {
    log.info("Unknown topic or partition {} {}", topic, partition);
    return null;
  }
  throw new IllegalStateException(
      "Error reading offset for " + topic + " / " + partition + ": " +
      ErrorMapping.exceptionNameFor(errorCode));
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
/**
 * Defines where to start reading data from
 * Helpers Available:
 * kafka.api.OffsetRequest.EarliestTime() => finds the beginning of the data in the logs and starts streaming
 * from there
 * kafka.api.OffsetRequest.LatestTime()   => will only stream new messages
 *
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
/***
     * Get the offset of the last message in the specified topic and partition
     *
     * @param consumer   consumer object
     * @param topic      topic name
     * @param partition  partition id
     * @param whichTime  time
     * @param clientName client name
     * @return offset value
     * @throws Exception if leader is not found or there is an error fetching data offset from broker.
     */
    public static long getLastOffset(kafka.javaapi.consumer.SimpleConsumer consumer, String topic, int partition,
                                     long whichTime, String clientName) throws Exception {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
        OffsetResponse response = consumer.getOffsetsBefore(request);

        if (response.hasError()) {
            throw new Exception("Error fetching data Offset from the Broker. Reason: " + response.errorCode(topic, partition));
//            return 0;
        }

        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
		List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
		Node leader,
		ExceptionProxy errorHandler) throws IOException, ClassNotFoundException {
	// each thread needs its own copy of the deserializer, because the deserializer is
	// not necessarily thread safe
	final KeyedDeserializationSchema<T> clonedDeserializer =
			InstantiationUtil.clone(deserializer, runtimeContext.getUserCodeClassLoader());

	// seed thread with list of fetch partitions (otherwise it would shut down immediately again
	SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
			this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue,
			clonedDeserializer, invalidOffsetBehavior);

	brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
			runtimeContext.getTaskName(), leader.id(), leader.host(), leader.port()));
	brokerThread.setDaemon(true);
	brokerThread.start();

	LOG.info("Starting thread {}", brokerThread.getName());
	return brokerThread;
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private void getMissingOffsetsFromKafka(
		List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws IOException
{
	// collect which partitions we should fetch offsets for
	List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToGetOffsetsFor = new ArrayList<>();
	for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) {
		if (!part.isOffsetDefined()) {
			// retrieve the offset from the consumer
			partitionsToGetOffsetsFor.add(part);
		}
	}
	
	if (partitionsToGetOffsetsFor.size() > 0) {
		getLastOffsetFromKafka(consumer, partitionsToGetOffsetsFor, invalidOffsetBehavior);
		
		LOG.info("No checkpoint/savepoint offsets found for some partitions. " +
				"Fetched the following start offsets {}", partitionsToGetOffsetsFor);
	}
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
@Override
public void commitInternalOffsetsToKafka(Map<KafkaTopicPartition, Long> offsets) throws Exception {
	ZookeeperOffsetHandler zkHandler = this.zookeeperOffsetHandler;
	if (zkHandler != null) {
		// the ZK handler takes care of incrementing the offsets by 1 before committing
		zkHandler.prepareAndCommitOffsets(offsets);
	}

	// Set committed offsets in topic partition state
	KafkaTopicPartitionState<TopicAndPartition>[] partitions = subscribedPartitions();
	for (KafkaTopicPartitionState<TopicAndPartition> partition : partitions) {
		Long offset = offsets.get(partition.getKafkaTopicPartition());
		if (offset != null) {
			partition.setCommittedOffset(offset);
		}
	}
}
 

import kafka.common.TopicAndPartition; //导入依赖的package包/类
private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
		List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
		Node leader,
		ExceptionProxy errorHandler) throws IOException, ClassNotFoundException
{
	// each thread needs its own copy of the deserializer, because the deserializer is
	// not necessarily thread safe
	final KeyedDeserializationSchema<T> clonedDeserializer =
			InstantiationUtil.clone(deserializer, runtimeContext.getUserCodeClassLoader());

	// seed thread with list of fetch partitions (otherwise it would shut down immediately again
	SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
			this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue, 
			clonedDeserializer, invalidOffsetBehavior);

	brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
			runtimeContext.getTaskName(), leader.id(), leader.host(), leader.port()));
	brokerThread.setDaemon(true);
	brokerThread.start();

	LOG.info("Starting thread {}", brokerThread.getName());
	return brokerThread;
}