Java TopicMetadataRequest类代码示例

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < NUM_TRIES_FETCH_TOPIC; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public static PartitionMetadata getPartitionMetadata(final SimpleConsumer consumer, final List<String> topics, final int partitionId) {
    try {
        TopicMetadataRequest req = new TopicMetadataRequest(topics);
        TopicMetadataResponse resp = consumer.send(req);

        List<TopicMetadata> topicMetadataList = resp.topicsMetadata();

        for (TopicMetadata metaData : topicMetadataList) {
            for (PartitionMetadata part : metaData.partitionsMetadata()) {
                if (part.partitionId() == partitionId) {
                    return part;
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Unable to fetch partition meta data from host[{}:{}] [{}:{}]", consumer.host(), consumer.port(), topics, partitionId, e);
    }

    return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private void updateLeaderMap() {
  for (String broker : brokerList) {
    try {
      SimpleConsumer consumer = getSimpleConsumer(broker);
      TopicMetadataRequest req = new TopicMetadataRequest(auditTopics);
      kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
      List<TopicMetadata> metaData = resp.topicsMetadata();

      for (TopicMetadata tmd : metaData) {
        for (PartitionMetadata pmd : tmd.partitionsMetadata()) {
          TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId());
          partitionLeader.put(topicAndPartition, getHostPort(pmd.leader()));
        }
      }
    } catch (Exception e) {
      logger.warn("Got exception to get metadata from broker=" + broker, e);
    }
  }
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public boolean doesTopicExist(String topic) {
    log.debug("Does Topic {} exist?", topic);
    SimpleConsumer consumer = new SimpleConsumer(host, port, soTimeout, bufferSize, clientId);
    List<String> topics = new ArrayList<>();
    TopicMetadataRequest request = new TopicMetadataRequest(topics);
    TopicMetadataResponse response = consumer.send(request);
    List<TopicMetadata> metadata = response.topicsMetadata();

    for (TopicMetadata item : metadata) {
        if (item.topic().equals(topic)) {
            log.debug("Found Topic {}.", topic);
            return true;
        }
    }
    log.debug("Did not find Topic {}.", topic);
    return false;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  log.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public int getNumPartitions(String topic) {
    SimpleConsumer consumer = null;
    try {
        consumer = createConsumer(
            mConfig.getKafkaSeedBrokerHost(),
            mConfig.getKafkaSeedBrokerPort(),
            "partitionLookup");
        List<String> topics = new ArrayList<String>();
        topics.add(topic);
        TopicMetadataRequest request = new TopicMetadataRequest(topics);
        TopicMetadataResponse response = consumer.send(request);
        if (response.topicsMetadata().size() != 1) {
            throw new RuntimeException("Expected one metadata for topic " + topic + " found " +
                response.topicsMetadata().size());
        }
        TopicMetadata topicMetadata = response.topicsMetadata().get(0);
        return topicMetadata.partitionsMetadata().size();
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public TopicMetadata findTopicMetadata(final String topic, int retries) throws Exception {
  Operation<TopicMetadata> findTopicOperation = new Operation<TopicMetadata>() {
    @Override
    public TopicMetadata execute() throws Exception {
      List<String> topics = Collections.singletonList(topic);
      TopicMetadataRequest req = new TopicMetadataRequest(topics);
      TopicMetadataResponse resp = consumer.send(req);

      List<TopicMetadata> topicMetadatas = resp.topicsMetadata();
      if (topicMetadatas.size() != 1) {
        throw new Exception("Expect to find 1 topic " + topic + ", but found " + topicMetadatas.size());
      }

      return topicMetadatas.get(0);
    }
  };
  return execute(findTopicOperation, retries);
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
/** 根据Topic列表返回TopicMetaData信息
 * 
 * @param topics
 * @return */
public static List<TopicMetadata> getTopicMetaData(List<String> topics) {
	SimpleConsumer simpleConsumer = SimpleKafkaHelper.getDefaultSimpleConsumer();
	TopicMetadataRequest metaDataRequest = new TopicMetadataRequest(topics);
	TopicMetadataResponse resp = simpleConsumer.send(metaDataRequest);
	List<TopicMetadata> metadatas = resp.topicsMetadata();

	return metadatas;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public List<String> listTopics() {
    log.debug("List Topics");
    SimpleConsumer consumer = new SimpleConsumer(host, port, soTimeout, bufferSize, clientId);
    List<String> topics = new ArrayList<>();
    TopicMetadataRequest request = new TopicMetadataRequest(topics);
    TopicMetadataResponse response = consumer.send(request);
    List<TopicMetadata> metadata = response.topicsMetadata();

    for (TopicMetadata item : metadata) {
        topics.add(item.topic());
    }

    log.debug("Found {} Topics", topics.size());
    return topics;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
/**
 * Gets the metadata from Kafka
 * 
 * @param conf
 * @return
 */
public List<TopicMetadata> getKafkaMetadata(JobConf conf) {
	ArrayList<String> metaRequestTopics = new ArrayList<String>();
	String brokerString = getKafkaBrokers(conf);
	if (brokerString.isEmpty())
		throw new InvalidParameterException("kafka.brokers must contain at least one node");
               List<String> brokers = Arrays.asList(brokerString.split("\\s*,\\s*"));
	Collections.shuffle(brokers);
	boolean fetchMetaDataSucceeded = false;
	int i = 0;
	List<TopicMetadata> topicMetadataList = null;
	Exception savedException = null;
	while (i < brokers.size() && !fetchMetaDataSucceeded) {
     log.info("Trying to connect to broker: " + brokers.get(i));
		SimpleConsumer consumer = createConsumer(conf, brokers.get(i));
		log.info(String.format("Fetching metadata from broker %s with client id %s for %d topic(s) %s",
		brokers.get(i), consumer.clientId(), metaRequestTopics.size(), metaRequestTopics));
		try {
			topicMetadataList = consumer.send(new TopicMetadataRequest(metaRequestTopics)).topicsMetadata();
			fetchMetaDataSucceeded = true;
		} catch (Exception e) {
			savedException = e;
			log.warn(String.format("Fetching topic metadata with client id %s for topics [%s] from broker [%s] failed",
				consumer.clientId(), metaRequestTopics, brokers.get(i)), e);
		} finally {
			consumer.close();
			i++;
		}
	}
	if (!fetchMetaDataSucceeded) {
		throw new RuntimeException("Failed to obtain metadata!", savedException);
	}
	return topicMetadataList;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
public static List<kafka.javaapi.TopicMetadata> MetadataDump(){
  kafka.javaapi.consumer.SimpleConsumer consumer = new SimpleConsumer("0.0.0.0", 9092, 100000, 64 * 1024, "metadata");
  List<String> topics = new ArrayList<String>();
  TopicMetadataRequest req = new TopicMetadataRequest(topics);
  kafka.javaapi.TopicMetadataResponse res = consumer.send(req);
  return res.topicsMetadata();
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private HostAndPort findLeader(TopicPartition topicPartition) {
    SimpleConsumer consumer = null;
    try {
        LOG.debug("looking up leader for topic {} partition {}", topicPartition.getTopic(), topicPartition.getPartition());
        consumer = createConsumer(
            mConfig.getKafkaSeedBrokerHost(),
            mConfig.getKafkaSeedBrokerPort(),
            "leaderLookup");
        List<String> topics = new ArrayList<String>();
        topics.add(topicPartition.getTopic());
        TopicMetadataRequest request = new TopicMetadataRequest(topics);
        TopicMetadataResponse response = consumer.send(request);

        List<TopicMetadata> metaData = response.topicsMetadata();
        for (TopicMetadata item : metaData) {
            for (PartitionMetadata part : item.partitionsMetadata()) {
                if (part.partitionId() == topicPartition.getPartition()) {
                    return HostAndPort.fromParts(part.leader().host(), part.leader().port());
                }
            }
        }
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
    return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
@Override
public Integer getNumPartitionsForTopic(String topic){
    TopicMetadataRequest topicRequest = new TopicMetadataRequest(Arrays.asList(topic));
    TopicMetadataResponse topicResponse = simpleConsumer.send(topicRequest);
    for (TopicMetadata topicMetadata : topicResponse.topicsMetadata()) {
        if (topic.equals(topicMetadata.topic())) {
            int partitionSize = topicMetadata.partitionsMetadata().size();
            logger.debug("Partition size found ({}) for {} topic", partitionSize, topic);
            return partitionSize;
        }
    }
    logger.warn("Metadata info not found!. TOPIC {}", topic);
    return null;
}
 

import kafka.javaapi.TopicMetadataRequest; //导入依赖的package包/类
private static TopicMetadata getTopicMetaData(SimpleConsumer consumer, String topic) {
  TopicMetadataRequest req = new TopicMetadataRequest(Lists.newArrayList(topic));

  List<TopicMetadata> metadatas = consumer.send(req).topicsMetadata();
  if (metadatas.size() != 1 || !metadatas.get(0).topic().equals(topic)) {
    LOG.fatal("no valid topic metadata for topic:", topic, ". run kafka-list-topic.sh to verify");
    System.exit(1);
  }
  return metadatas.get(0);
}