Java Cluster类代码示例

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
/**
 * Refresh the metadata. The method is synchronized because the network client is not thread safe.
 */
public synchronized ClusterAndGeneration refreshMetadata(long timeout) {
  // Do not update metadata if the metadata has just been refreshed.
  if (_time.milliseconds() >= _metadata.lastSuccessfulUpdate() + _metadataTTL) {
    // Cruise Control always fetch metadata for all the topics.
    _metadata.needMetadataForAllTopics(true);
    int version = _metadata.requestUpdate();
    long remaining = timeout;
    Cluster beforeUpdate = _metadata.fetch();
    while (_metadata.version() <= version && remaining > 0) {
      _metadata.requestUpdate();
      long start = _time.milliseconds();
      _networkClient.poll(remaining, start);
      remaining -= (_time.milliseconds() - start);
    }
    LOG.debug("Updated metadata {}", _metadata.fetch());
    if (MonitorUtils.metadataChanged(beforeUpdate, _metadata.fetch())) {
      _metadataGeneration.incrementAndGet();
    }
  }
  return new ClusterAndGeneration(_metadata.fetch(), _metadataGeneration.get());
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
TrainingFetcher(MetricSampler metricSampler,
                Cluster cluster,
                SampleStore sampleStore,
                Set<TopicPartition> assignedPartitions,
                long startTimeMs,
                long endTimeMs,
                Timer fetcherTimer,
                Meter fetcherFailureRate) {
  _cluster = cluster;
  _sampleStore = sampleStore;
  _metricSampler = metricSampler;
  _startTimeMs = startTimeMs;
  _endTimeMs = endTimeMs;
  _assignedPartitions = assignedPartitions;
  _fetcherTimer = fetcherTimer;
  _fetcherFailureRate = fetcherFailureRate;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
@Override
public List<Set<TopicPartition>> assignPartitions(Cluster cluster, int numMetricFetchers) {
  // Create an array to host the assignment of all the metric fetchers.
  List<Set<TopicPartition>> assignments = new ArrayList<>();
  for (int i = 0; i < numMetricFetchers; i++) {
    assignments.add(new HashSet<>());
  }
  int index = 0;
  // The total number of partitions that has been assigned.
  int totalPartitionAssigned = 0;
  for (String topic : cluster.topics()) {
    while (assignments.get(index % numMetricFetchers).size() > totalPartitionAssigned / numMetricFetchers) {
      index++;
    }
    Set<TopicPartition> assignmentForFetcher = assignments.get(index % numMetricFetchers);
    List<PartitionInfo> partitionsForTopic = cluster.partitionsForTopic(topic);
    for (PartitionInfo partitionInfo : partitionsForTopic) {
      assignmentForFetcher.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
    }
    totalPartitionAssigned += partitionsForTopic.size();
  }
  // Print the assignments if the logger is set to debug level or lower.
  maybeDumpAssignments(assignments);
  return assignments;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
SamplingFetcher(MetricSampler metricSampler,
                Cluster cluster,
                MetricSampleAggregator metricSampleAggregator,
                SampleStore sampleStore,
                Set<TopicPartition> assignedPartitions,
                long startTimeMs,
                long endTimeMs,
                boolean leaderValidation,
                boolean useLinearRegressionModel,
                Timer fetchTimer,
                Meter fetchFailureRate) {
  _metricSampler = metricSampler;
  _cluster = cluster;
  _metricSampleAggregator = metricSampleAggregator;
  _sampleStore = sampleStore;
  _assignedPartitions = assignedPartitions;
  _startTimeMs = startTimeMs;
  _endTimeMs = endTimeMs;
  _leaderValidation = leaderValidation;
  _useLinearRegressionModel = useLinearRegressionModel;
  _fetchTimer = fetchTimer;
  _fetchFailureRate = fetchFailureRate;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
/**
 * Get the number of valid windows that meets the minimum monitored partitions percentage requirement.
 *
 * @param minMonitoredPartitionsPercentage the minimum monitored partitions percentage.
 * @param totalNumPartitions the total number of partitions.
 * @return the number of the most recent valid windows.
 */
synchronized public int numValidWindows(ModelGeneration modelGeneration,
                                        Cluster cluster,
                                        double minMonitoredPartitionsPercentage,
                                        int totalNumPartitions) {
  updateMetricCompleteness(cluster, modelGeneration);
  int i = 0;
  double minMonitoredNumPartitions = totalNumPartitions * minMonitoredPartitionsPercentage;
  for (Map.Entry<Long, Integer> entry : _validPartitionsByWindows.entrySet()) {
    long window = entry.getKey();
    if ((entry.getValue() < minMonitoredNumPartitions && window != _activeSnapshotWindow) || i == _maxNumSnapshots) {
      break;
    }
    if (window != _activeSnapshotWindow) {
      i++;
    }
  }
  return i;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
private Snapshot avgAdjacentSnapshots(TopicPartition tp,
                                      Cluster cluster,
                                      long snapshotWindow) {
  // The aggregated metrics for the current window should never be null. We have checked that earlier.
  AggregatedMetrics aggMetricsForPartition = _windowedAggregatedPartitionMetrics.get(snapshotWindow).get(tp);
  Snapshot currSnapshot =
      aggMetricsForPartition == null ? null : aggMetricsForPartition.toSnapshot(snapshotWindow);
  Snapshot prevSnapshot = partitionSnapshotForWindow(tp, cluster, snapshotWindow - _snapshotWindowMs,
                                                     snapshotWindow - _snapshotWindowMs, false, false).snapshot();
  Snapshot nextSnapshot = partitionSnapshotForWindow(tp, cluster, snapshotWindow + _snapshotWindowMs,
                                                     snapshotWindow + _snapshotWindowMs, false, false).snapshot();
  if (prevSnapshot != null && nextSnapshot != null) {
    return new Snapshot(snapshotWindow,
                        averageNullableUtilization(Resource.CPU, prevSnapshot, currSnapshot, nextSnapshot),
                        averageNullableUtilization(Resource.NW_IN, prevSnapshot, currSnapshot, nextSnapshot),
                        averageNullableUtilization(Resource.NW_OUT, prevSnapshot, currSnapshot, nextSnapshot),
                        averageNullableUtilization(Resource.DISK, prevSnapshot, currSnapshot, nextSnapshot));
  } else {
    return null;
  }
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
private boolean isReplicaMovementDone(Cluster cluster, TopicPartition tp, ExecutionTask task) {
  boolean destinationExists = false;
  boolean sourceExists = false;
  for (Node node : cluster.partition(tp).replicas()) {
    destinationExists = destinationExists || (node.id() == task.destinationBrokerId());
    sourceExists = sourceExists || (node.id() == task.sourceBrokerId());
  }
  switch (task.state()) {
    case IN_PROGRESS:
      return destinationExists && !sourceExists;
    case ABORTING:
      return !destinationExists && sourceExists;
    case DEAD:
      return !destinationExists && !sourceExists;
    default:
      throw new IllegalStateException("Should never be here. State " + task.state());
  }
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
private Metadata getMetadata(Collection<TopicPartition> partitions) {
  Node node0 = new Node(0, "localhost", 100, "rack0");
  Node node1 = new Node(1, "localhost", 100, "rack1");
  Node[] nodes = {node0, node1};
  Set<Node> allNodes = new HashSet<>();
  allNodes.add(node0);
  allNodes.add(node1);
  Set<PartitionInfo> parts = new HashSet<>();
  for (TopicPartition tp : partitions) {
    parts.add(new PartitionInfo(tp.topic(), tp.partition(), node0, nodes, nodes));
  }
  Cluster cluster = new Cluster("cluster-id", allNodes, parts, Collections.emptySet(), Collections.emptySet());
  Metadata metadata = new Metadata();
  metadata.update(cluster, Collections.emptySet(), 0);
  return metadata;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
private Metadata getMetadata(Collection<TopicPartition> partitions) {
  Node node0 = new Node(0, "localhost", 100, "rack0");
  Node node1 = new Node(1, "localhost", 100, "rack1");
  Node[] nodes = {node0, node1};
  Set<Node> allNodes = new HashSet<>();
  allNodes.add(node0);
  allNodes.add(node1);
  Set<PartitionInfo> parts = new HashSet<>();
  for (TopicPartition tp : partitions) {
    parts.add(new PartitionInfo(tp.topic(), tp.partition(), node0, nodes, nodes));
  }
  Cluster cluster = new Cluster("clusterId", allNodes, parts, Collections.emptySet(), Collections.emptySet());
  Metadata metadata = new Metadata();
  metadata.update(cluster, Collections.emptySet(), 0);
  return metadata;
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
@Override
public Samples getSamples(Cluster cluster,
                          Set<TopicPartition> assignedPartitions,
                          long startTime,
                          long endTime,
                          SamplingMode mode) throws MetricSamplingException {

  if (_exceptionsLeft > 0) {
    _exceptionsLeft--;
    throw new MetricSamplingException("Error");
  }
  Set<PartitionMetricSample> partitionMetricSamples = new HashSet<>();
  for (TopicPartition tp: assignedPartitions) {
    PartitionMetricSample sample = new PartitionMetricSample(cluster.partition(tp).leader().id(), tp);
    long now = TIME.milliseconds();
    for (Resource resource : Resource.values()) {
      sample.record(resource, now);
    }
    sample.close(now);
    partitionMetricSamples.add(sample);
  }

  return new Samples(partitionMetricSamples, Collections.emptySet());
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
/**
 * Get a snapshot of the cluster metadata from this response
 * @return the cluster snapshot
 */
public Cluster cluster() {
    Set<String> internalTopics = new HashSet<>();
    List<PartitionInfo> partitions = new ArrayList<>();
    for (TopicMetadata metadata : topicMetadata) {
        if (metadata.error == Errors.NONE) {
            if (metadata.isInternal)
                internalTopics.add(metadata.topic);
            for (PartitionMetadata partitionMetadata : metadata.partitionMetadata)
                partitions.add(new PartitionInfo(
                        metadata.topic,
                        partitionMetadata.partition,
                        partitionMetadata.leader,
                        partitionMetadata.replicas.toArray(new Node[0]),
                        partitionMetadata.isr.toArray(new Node[0])));
        }
    }

    return new Cluster(this.clusterId, this.brokers, partitions, topicsByError(Errors.TOPIC_AUTHORIZATION_FAILED),
            internalTopics, this.controller);
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
/**
 * 0.10.x clients can't talk with 0.9.x brokers, and 0.10.0.0 introduced the new protocol with API versions.
 * That means we can simulate an API version mismatch.
 *
 * @throws Exception
 */
@Test
public void returnNullWithApiVersionMismatch() {
    final NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    boolean internal = false;
    Cluster cluster = createCluster(1);
    try (MockKafkaAdminClientEnv env = new MockKafkaAdminClientEnv(cluster)) {
        env.kafkaClient().setNode(cluster.controller());
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.<String>emptySet());
        env.kafkaClient().prepareResponse(createTopicResponseWithUnsupportedVersion(newTopic));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        admin.createTopic(newTopic);
        fail();
    } catch (UnsupportedVersionException e) {
        // expected
    }
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
@Before
public void setupCoordinator() {
    this.mockTime = new MockTime();
    this.mockClient = new MockClient(mockTime);

    Metadata metadata = new Metadata(100L, 60 * 60 * 1000L, true);
    this.consumerClient = new ConsumerNetworkClient(mockClient, metadata, mockTime,
            RETRY_BACKOFF_MS, REQUEST_TIMEOUT_MS);
    Metrics metrics = new Metrics();

    Cluster cluster = TestUtils.singletonCluster("topic", 1);
    metadata.update(cluster, Collections.<String>emptySet(), mockTime.milliseconds());
    this.node = cluster.nodes().get(0);
    mockClient.setNode(node);

    this.coordinatorNode = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    this.coordinator = new DummyCoordinator(consumerClient, metrics, mockTime);
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
@Test
public void shouldCreateOneTopicWhenProvidedMultipleDefinitionsWithSameTopicName() {
    NewTopic newTopic1 = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    NewTopic newTopic2 = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (MockKafkaAdminClientEnv env = new MockKafkaAdminClientEnv(cluster)) {
        env.kafkaClient().setNode(cluster.controller());
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.<String>emptySet());
        env.kafkaClient().prepareResponse(createTopicResponse(newTopic1));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        Set<String> newTopicNames = admin.createTopics(newTopic1, newTopic2);
        assertEquals(1, newTopicNames.size());
        assertEquals(newTopic2.name(), newTopicNames.iterator().next());
    }
}
 

import org.apache.kafka.common.Cluster; //导入依赖的package包/类
@Test
public void shouldSetClusterMetadataOnAssignment() throws Exception {
    final List<TopicPartition> topic = Collections.singletonList(new TopicPartition("topic", 0));
    final Map<HostInfo, Set<TopicPartition>> hostState =
            Collections.singletonMap(new HostInfo("localhost", 80),
                    Collections.singleton(new TopicPartition("topic", 0)));
    final AssignmentInfo assignmentInfo = new AssignmentInfo(Collections.singletonList(new TaskId(0, 0)),
            Collections.<TaskId, Set<TopicPartition>>emptyMap(),
            hostState);


    partitionAssignor.onAssignment(new PartitionAssignor.Assignment(topic, assignmentInfo.encode()));
    final Cluster cluster = partitionAssignor.clusterMetadata();
    final List<PartitionInfo> partitionInfos = cluster.partitionsForTopic("topic");
    final PartitionInfo partitionInfo = partitionInfos.get(0);
    assertEquals(1, partitionInfos.size());
    assertEquals("topic", partitionInfo.topic());
    assertEquals(0, partitionInfo.partition());
}