Java Cluster.partitionCountForTopic方法代码示例

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
@Override
public Map<String, Assignment> assign(Cluster metadata, Map<String, Subscription> subscriptions) {
    Set<String> allSubscribedTopics = new HashSet<>();
    // 解析subscriptionEntry，去除userData信息
    for (Map.Entry<String, Subscription> subscriptionEntry : subscriptions.entrySet())
        allSubscribedTopics.addAll(subscriptionEntry.getValue().topics());

    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    // 统计每个topic分区的数量
    for (String topic : allSubscribedTopics) {
        Integer numPartitions = metadata.partitionCountForTopic(topic);
        if (numPartitions != null && numPartitions > 0)
            partitionsPerTopic.put(topic, numPartitions);
        else
            log.debug("Skipping assignment for topic {} since no metadata is available", topic);
    }
    //留给实现类去实现
    Map<String, List<TopicPartition>> rawAssignments = assign(partitionsPerTopic, subscriptions);

    // this class maintains no user data, so just wrap the results
    Map<String, Assignment> assignments = new HashMap<>();
    // 整理分区分配的结果
    for (Map.Entry<String, List<TopicPartition>> assignmentEntry : rawAssignments.entrySet())
        assignments.put(assignmentEntry.getKey(), new Assignment(assignmentEntry.getValue()));
    return assignments;
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
/**
 * Wait for cluster metadata including partitions for the given topic to be available.
 *
 * @param topic     The topic we want metadata for
 * @param partition A specific partition expected to exist in metadata, or null if there's no preference
 * @param maxWaitMs The maximum time in ms for waiting on the metadata
 * @return The cluster containing topic metadata and the amount of time we waited in ms
 */
// 唤醒sender线程更新metadata中保存的kafka集群元数据
private ClusterAndWaitTime waitOnMetadata(String topic, Integer partition, long maxWaitMs) throws InterruptedException {
    // add topic to metadata topic list if it is not there already and reset expiry
    metadata.add(topic);
    Cluster cluster = metadata.fetch();

    // 判断是不是包含指定的topic
    Integer partitionsCount = cluster.partitionCountForTopic(topic);
    // Return cached metadata if we have it, and if the record's partition is either undefined
    // or within the known partition range
    if (partitionsCount != null && (partition == null || partition < partitionsCount))
        return new ClusterAndWaitTime(cluster, 0);

    long begin = time.milliseconds();
    long remainingWaitMs = maxWaitMs;
    long elapsed;
    // Issue metadata requests until we have metadata for the topic or maxWaitTimeMs is exceeded.
    // In case we already have cached metadata for the topic, but the requested partition is greater
    // than expected, issue an update request only once. This is necessary in case the metadata
    // is stale and the number of partitions for this topic has increased in the meantime.
    do {
        log.trace("Requesting metadata update for topic {}.", topic);
        metadata.add(topic);
        // 获取当前元数据版本号
        int version = metadata.requestUpdate();
        // 唤醒send线程
        sender.wakeup();
        try {
            // 阻塞元数据等待更新完成
            metadata.awaitUpdate(version, remainingWaitMs);
        } catch (TimeoutException ex) {
            // Rethrow with original maxWaitMs to prevent logging exception with remainingWaitMs
            throw new TimeoutException("Failed to update metadata after " + maxWaitMs + " ms.");
        }
        cluster = metadata.fetch();
        elapsed = time.milliseconds() - begin;
        // 检测超时时间
        if (elapsed >= maxWaitMs)
            throw new TimeoutException("Failed to update metadata after " + maxWaitMs + " ms.");
        // 检测权限
        if (cluster.unauthorizedTopics().contains(topic))
            throw new TopicAuthorizationException(topic);
        remainingWaitMs = maxWaitMs - elapsed;
        partitionsCount = cluster.partitionCountForTopic(topic);
    } while (partitionsCount == null);

    if (partition != null && partition >= partitionsCount) {
        throw new KafkaException(
                String.format("Invalid partition given with record: %d is not in the range [0...%d).", partition, partitionsCount));
    }

    return new ClusterAndWaitTime(cluster, elapsed);
}
 

import org.apache.kafka.common.Cluster; //导入方法依赖的package包/类
/**
 * Compute the partition for the given record.
 *
 * @param topic      The topic name
 * @param key        The key to partition on (or null if no key)
 * @param keyBytes   The serialized key to partition on( or null if no key)
 * @param value      The value to partition on or null
 * @param valueBytes The serialized value to partition on or null
 * @param cluster    The current cluster metadata
 */
@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster)
{
    int partitionNum = cluster.partitionCountForTopic(topic);
    return partitionByMod((Long) key, partitionNum);
}