Java PartitionMetadata.leader方法代码示例

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private static OffsetInfo getOffset(String topic, PartitionMetadata partition) {
  Broker broker = partition.leader();

  SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 10000, 1000000,
                                               "com.rekko.newrelic.storm.kafka");
  try {
    TopicAndPartition
        topicAndPartition =
        new TopicAndPartition(topic, partition.partitionId());
    PartitionOffsetRequestInfo rquest = new PartitionOffsetRequestInfo(-1, 1);
    Map<TopicAndPartition, PartitionOffsetRequestInfo>
        map =
        new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    map.put(topicAndPartition, rquest);
    OffsetRequest req = new OffsetRequest(map, (short) 0, "com.rekko.newrelic.storm.kafka");
    OffsetResponse resp = consumer.getOffsetsBefore(req);
    OffsetInfo offset = new OffsetInfo();
    offset.offset = resp.offsets(topic, partition.partitionId())[0];
    return offset;
  } finally {
    consumer.close();
  }
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private String findNewLeader(KafkaBrokerInfo brokerInfo, String topic, int partition) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(brokerInfo, topic, partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private String findNewLeader(List<KafkaBrokerInfo> brokerInfoList, String topic, int partition) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(brokerInfoList, topic, partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            //
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    LOG.info("Unable to find new leader after Broker failure. Exiting");
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private HostAndPort findNewLeader(HostAndPort oldLeader, String topic, int partition) throws StageException {
  //try 3 times to find a new leader
  for (int i = 0; i < 3; i++) {
    boolean sleep;
    PartitionMetadata metadata = getPartitionMetadata(replicaBrokers, topic, partition);
    if (metadata == null || metadata.leader() == null) {
      sleep = true;
    } else if (oldLeader.getHostText().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
      //leader has not yet changed, give zookeeper sometime
      sleep = true;
    } else {
      return HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
    }
    if (sleep) {
      ThreadUtil.sleep(ONE_SECOND);
    }
  }
  LOG.error(KafkaErrors.KAFKA_21.getMessage());
  throw new StageException(KafkaErrors.KAFKA_21);
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            //
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    System.out.println("Unable to find new leader after Broker failure. Exiting");
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private Broker findNewLeader(Broker oldLeader) throws InterruptedException {
    long retryCnt = 0;
    while (true) {
        PartitionMetadata metadata = findLeader();
        logger.debug("findNewLeader - meta leader {}, previous leader {}", metadata, oldLeader);
        if (metadata != null && metadata.leader() != null && (oldLeader == null ||
                (!(oldLeader.host().equalsIgnoreCase(metadata.leader().host()) &&
                  (oldLeader.port() == metadata.leader().port())) || retryCnt != 0))) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            logger.info("findNewLeader - using new leader {} from meta data, previous leader {}", metadata.leader(), oldLeader);
            return metadata.leader();
        }
        //TODO: backoff retry
        Thread.sleep(1000L);
        retryCnt ++;
        // if could not find the leader for current replicaBrokers, let's try to find one via allBrokers
        if (retryCnt >= 3 && (retryCnt - 3) % 5 == 0) {
            logger.warn("can nof find leader for {} - {} after {} retries", topic, partitionId, retryCnt);
            replicaBrokers.clear();
            replicaBrokers.addAll(allBrokers);
        }
    }
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
  List<KafkaPartition> partitions = Lists.newArrayList();

  for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
    if (null == partitionMetadata) {
      LOG.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
      return Collections.emptyList();
    }
    if (null == partitionMetadata.leader()) {
      LOG.error(
          "Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada=" + partitionMetadata);
      return Collections.emptyList();
    }
    partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
        .withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
        .withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
  }
  return partitions;
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
  List<KafkaPartition> partitions = Lists.newArrayList();

  for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
    if (null == partitionMetadata) {
      log.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
      return Collections.emptyList();
    }
    if (null == partitionMetadata.leader()) {
      log.error("Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada="
          + partitionMetadata);
      return Collections.emptyList();
    }
    partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
        .withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
        .withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
  }
  return partitions;
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private HostPort findNewLeader(String oldHost, int oldPort) throws LostLeadershipException {
  for (int i = 0; i < 3; i++) {
    boolean goToSleep = false;
    PartitionMetadata metadata = findLeader(replicaBrokers, topic, partition);
    if (metadata == null) {
      goToSleep = true;
    } else if (metadata.leader() == null) {
      goToSleep = true;
    } else if (oldHost.equalsIgnoreCase(metadata.leader().host()) &&
        oldPort == metadata.leader().port()) {
      // first time through if the leader hasn't changed give ZooKeeper a second to recover
      // second time, assume the broker did recover before failover, or it was a non-Broker issue
      goToSleep = true;
    } else {
      return new HostPort(metadata.leader().host(), metadata.leader().port());
    }
    if (goToSleep) {
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
      }
    }
  }
  // Can't recover from a leadership disappearance.
  throw new LostLeadershipException();
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
public void run() {
  try {
    while (!exit) {
      KafkaTool kafkaTool = new KafkaTool(topic, cluster.getZKConnect());
      kafkaTool.connect();
      TopicMetadata topicMeta = kafkaTool.findTopicMetadata(topic);
      PartitionMetadata partitionMeta = findPartition(topicMeta, partition);
      Broker partitionLeader = partitionMeta.leader();
      Server kafkaServer = cluster.findKafkaServerByPort(partitionLeader.port());
      System.out.println("Shutdown kafka server " + kafkaServer.getPort());
      kafkaServer.shutdown();
      failureCount++;
      Thread.sleep(sleepBeforeRestart);
      kafkaServer.start();
      kafkaTool.close();
      Thread.sleep(10000); //wait to make sure that the kafka server start
    }
  } catch (Exception e) {
    e.printStackTrace();
  }
  synchronized (this) {
    notify();
  }
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private PartitionMetadata findLeader(List<KafkaBrokerInfo> brokerInfoList, String topic, int partition) {
    PartitionMetadata returnMetaData = null;
    for (KafkaBrokerInfo brokerInfo : brokerInfoList) {
        PartitionMetadata part = findLeader(brokerInfo, topic, partition);
        if (part != null && part.leader() != null) {
            return part;
        }
    }
    return null;
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
/**
 * 验证分区元数据，如果验证失败，直接抛出IllegalArgumentException异常
 *
 * @param metadata
 */
private void validatePartitionMetadata(PartitionMetadata metadata) {
    if (metadata == null) {
        System.out.println("Can't find metadata for Topic and Partition. Exiting!!");
        throw new IllegalArgumentException("Can't find metadata for Topic and Partition. Exiting!!");
    }
    if (metadata.leader() == null) {
        System.out.println("Can't find Leader for Topic and Partition. Exiting!!");
        throw new IllegalArgumentException("Can't find Leader for Topic and Partition. Exiting!!");
    }
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
/**
 * 根据给定参数获取一个新leader的分区元数据信息
 *
 * @param oldLeader
 * @param topic
 * @param partitionID
 *
 * @return
 */
private PartitionMetadata findNewLeaderMetadata(String oldLeader, String topic, int partitionID) {
    KafkaTopicPartitionInfo topicPartitionInfo = new KafkaTopicPartitionInfo(topic, partitionID);
    List<KafkaBrokerInfo> brokerInfos = this.replicaBrokers.get(topicPartitionInfo);
    for (int i = 0; i < 3; i++) {
        boolean gotoSleep = false;
        PartitionMetadata metadata = this.findLeader(brokerInfos, topic, partitionID);
        if (metadata == null) {
            gotoSleep = true;
        } else if (metadata.leader() == null) {
            gotoSleep = true;
        } else if (oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            // leader切换过程中
            gotoSleep = true;
        } else {
            return metadata;
        }

        if (gotoSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                // nothings
            }
        }
    }

    System.out.println("Unable to find new leader after Broker failure. Exiting!!");
    throw new RuntimeException("Unable to find new leader after Broker failure. Exiting!!");
}
 

import kafka.javaapi.PartitionMetadata; //导入方法依赖的package包/类
private PartitionMetadata findLeader(List<KafkaBrokerInfo> brokerInfoList, String topic, int partition) {
    PartitionMetadata returnMetaData = null;
    for (KafkaBrokerInfo brokerInfo : brokerInfoList) {
        PartitionMetadata part = findLeader(brokerInfo.brokerHost, brokerInfo.brokerPort, topic, partition);
        if (part != null && part.leader() != null) {
            return part;
        }
    }
    return null;
}