Java Cluster.nodes方法代码示例

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
/**
 * Add empty load of all the partitions that exists in the current cluster but missing from the
 * metric aggregation result.
 */
private void fillInMissingPartitions(Map<TopicPartition, Snapshot[]> loadSnapshots,
                                            Cluster kafkaCluster,
                                            ClusterModel clusterModel) throws ModelInputException {
  // There must be at least one entry, otherwise there will be exception thrown earlier. So we don't need to
  // check if it has next
  Snapshot[] snapshotsForTimestamps = loadSnapshots.values().iterator().next();
  Snapshot[] emptyLoadSnapshots = new Snapshot[snapshotsForTimestamps.length];
  for (int i = 0; i < emptyLoadSnapshots.length; i++) {
    emptyLoadSnapshots[i] = new Snapshot(snapshotsForTimestamps[i].time());
  }
  for (Node node : kafkaCluster.nodes()) {
    for (PartitionInfo partitionInfo : kafkaCluster.partitionsForNode(node.id())) {
      TopicPartition tp = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
      if (!loadSnapshots.containsKey(tp)) {
        populateSnapshots(kafkaCluster, clusterModel, tp, emptyLoadSnapshots);
      }
    }
  }
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
private Cluster getClusterForCurrentTopics(Cluster cluster) {
    Set<String> unauthorizedTopics = new HashSet<>();
    Collection<PartitionInfo> partitionInfos = new ArrayList<>();
    List<Node> nodes = Collections.emptyList();
    Set<String> internalTopics = Collections.emptySet();
    Node controller = null;
    String clusterId = null;
    if (cluster != null) {
        clusterId = cluster.clusterResource().clusterId();
        internalTopics = cluster.internalTopics();
        unauthorizedTopics.addAll(cluster.unauthorizedTopics());
        unauthorizedTopics.retainAll(this.topics.keySet());

        for (String topic : this.topics.keySet()) {
            List<PartitionInfo> partitionInfoList = cluster.partitionsForTopic(topic);
            if (!partitionInfoList.isEmpty()) {
                partitionInfos.addAll(partitionInfoList);
            }
        }
        nodes = cluster.nodes();
        controller  = cluster.controller();
    }
    return new Cluster(clusterId, nodes, partitionInfos, unauthorizedTopics, internalTopics, controller);
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
/**
 * Check whether the metadata has changed.
 */
public static boolean metadataChanged(Cluster prev, Cluster curr) {
  // Broker has changed.
  Set<Node> prevNodeSet = new HashSet<>(prev.nodes());
  if (prevNodeSet.size() != curr.nodes().size()) {
    return true;
  }
  prevNodeSet.removeAll(curr.nodes());
  if (!prevNodeSet.isEmpty()) {
    return true;
  }
  // Topic has changed
  if (!prev.topics().equals(curr.topics())) {
    return true;
  }

  // partition has changed.
  for (String topic : prev.topics()) {
    if (!prev.partitionCountForTopic(topic).equals(curr.partitionCountForTopic(topic))) {
      return true;
    }
    for (PartitionInfo prevPartInfo : prev.partitionsForTopic(topic)) {
      PartitionInfo currPartInfo = curr.partition(new TopicPartition(prevPartInfo.topic(), prevPartInfo.partition()));
      if (leaderChanged(prevPartInfo, currPartInfo) || replicaListChanged(prevPartInfo, currPartInfo)) {
        return true;
      }
    }
  }
  return false;
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
private void fillInFollowerBytesInRate(Cluster cluster, Map<Integer, Map<String, Integer>> leaderDistributionStats) {
  synchronized (this) {
    if (!_brokerFollowerLoad.isEmpty()) {
      return;
    }
    for (Node node : cluster.nodes()) {
      _brokerFollowerLoad.putIfAbsent(node.id(), 0.0);
      BrokerLoad brokerLoad = _brokerLoad.get(node.id());
      if (brokerLoad == null) {
        // new broker?
        continue;
      }
      for (PartitionInfo partitionInfo : cluster.partitionsForNode(node.id())) {
        IOLoad topicIOLoad = brokerLoad.ioLoad(partitionInfo.topic(), partitionInfo.partition());
        if (topicIOLoad == null) {
          // The topic did not report any IO metric and the partition does not exist on the broker.
          LOG.debug("No IO load reported from broker {} for partition {}-{}",
                    node.id(), partitionInfo.topic(), partitionInfo.partition());
          continue;
        }
        int numLeadersOnBroker = leaderDistributionStats.get(node.id()).get(partitionInfo.topic());
        double partitionBytesIn = topicIOLoad.bytesIn() / numLeadersOnBroker;
        for (Node replica : partitionInfo.replicas()) {
          if (replica.id() != node.id()) {
            _brokerFollowerLoad.merge(replica.id(), partitionBytesIn, (v0, v1) -> v0 + v1);
          }
        }
      }
    }
  }
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
/**
 * A helper function to get the number of leader partitions for each topic on each broker. It is useful to
 * derive the partition level IO from the topic level IO on a broker.
 * TODO: create open source KIP to provide per partition IO metrics.
 */
private Map<Integer, Map<String, Integer>> leaderDistributionStats(Cluster cluster) {
  Map<Integer, Map<String, Integer>> stats = new HashMap<>();
  for (Node node : cluster.nodes()) {
    Map<String, Integer> numLeadersByTopic = new HashMap<>();
    stats.put(node.id(), numLeadersByTopic);
    for (PartitionInfo partitionInfo : cluster.partitionsForNode(node.id())) {
      numLeadersByTopic.merge(partitionInfo.topic(), 1, (v0, v1) -> v0 + v1);
    }
  }
  return stats;
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
MetricSampler.Samples process(Cluster cluster,
                              Collection<TopicPartition> partitions,
                              MetricSampler.SamplingMode samplingMode) {
  Map<Integer, Map<String, Integer>> leaderDistributionStats = leaderDistributionStats(cluster);
  fillInFollowerBytesInRate(cluster, leaderDistributionStats);
  //TODO: maybe need to skip the entire processing logic if broker load is not consistent.
  // Theoretically we should not move forward at all if a broker reported a different all topic bytes in from all
  // its resident replicas. However, it is not clear how often this would happen yet. At this point we still
  // continue process the other brokers. Later on if in practice all topic bytes in and the aggregation value is
  // rarely inconsistent we can just stop the sample generation when the this happens.
  _brokerLoad.values().forEach(BrokerLoad::validate);

  Set<PartitionMetricSample> partitionMetricSamples = new HashSet<>();
  Set<BrokerMetricSample> brokerMetricSamples = new HashSet<>();

  int skippedPartition = 0;
  if (samplingMode == MetricSampler.SamplingMode.ALL
      || samplingMode == MetricSampler.SamplingMode.PARTITION_METRICS_ONLY) {
    for (TopicPartition tp : partitions) {
      try {
        PartitionMetricSample sample = buildPartitionMetricSample(cluster, tp, leaderDistributionStats);
        if (sample != null) {
          LOG.debug("Added partition metrics sample for {}", tp);
          partitionMetricSamples.add(sample);
        } else {
          skippedPartition++;
        }
      } catch (Exception e) {
        LOG.error("Error building partition metric sample for " + tp, e);
        skippedPartition++;
      }
    }
  }

  int skippedBroker = 0;
  if (samplingMode == MetricSampler.SamplingMode.ALL
      || samplingMode == MetricSampler.SamplingMode.BROKER_METRICS_ONLY) {
    for (Node node : cluster.nodes()) {
      BrokerLoad brokerLoad = _brokerLoad.get(node.id());
      if (brokerLoad == null || !brokerLoad.isValid()) {
        // A new broker or broker metrics are not consistent.
        LOG.debug("Skip generating broker metric sample for broker {} because it does not have IO load metrics or "
                      + "the metrics are inconsistent.", node.id());
        skippedBroker++;
        continue;
      }
      double leaderCpuUtil = brokerLoad.cpuUtil();
      if (leaderCpuUtil > 0) {
        BrokerMetricSample brokerMetricSample =
            new BrokerMetricSample(node.id(),
                                   leaderCpuUtil,
                                   brokerLoad.bytesIn() / BYTES_IN_KB,
                                   brokerLoad.bytesOut() / BYTES_IN_KB,
                                   // The replication bytes in is only available from Kafka 0.11.0 and above.
                                   (brokerLoad.replicationBytesIn() > 0 ?
                                       brokerLoad.replicationBytesIn() : _brokerFollowerLoad.get(node.id())) / BYTES_IN_KB,
                                   brokerLoad.replicationBytesOut() / BYTES_IN_KB,
                                   brokerLoad.messagesInRate(),
                                   brokerLoad.produceRequestRate(),
                                   brokerLoad.consumerFetchRequestRate(),
                                   brokerLoad.followerFetchRequestRate(),
                                   brokerLoad.requestHandlerAvgIdlePercent(),
                                   -1.0,
                                   brokerLoad.allTopicsProduceRequestRate(),
                                   brokerLoad.allTopicsFetchRequestRate(),
                                   _maxMetricTimestamp);
        LOG.debug("Added broker metric sample for broker {}", node.id());
        brokerMetricSamples.add(brokerMetricSample);
      } else {
        skippedBroker++;
      }
    }
  }
  LOG.info("Generated {}{} partition metric samples and {}{} broker metric samples for timestamp {}",
           partitionMetricSamples.size(), skippedPartition > 0 ? "(" + skippedPartition + " skipped)" : "",
           brokerMetricSamples.size(), skippedBroker > 0 ? "(" + skippedBroker + " skipped)" : "",
           _maxMetricTimestamp);
  return new MetricSampler.Samples(partitionMetricSamples, brokerMetricSamples);
}