Java TopicPartition.topic方法代码示例

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
@Override
public void pauseTopic(TopicPartition tp, long offset, ControlMessage message) {

    String topic = message.payloadValue("topic", String.class);
    String tableName = message.payloadValue("TABLE_NAME", String.class);
    if (topic == null || topic.length() == 0) topic = tp.topic();
    if (!pausedTopics.containsKey(topic)) {
        consumer.pause(Arrays.asList(tp));
        TopicInfo topicInfo = TopicInfo.build(tp.topic(), tp.partition(), offset, tableName);
        pausedTopics.put(topic, topicInfo);

        try {
            zkNodeOperator.setData(pausedTopics, true);
        } catch (Exception e) {
            logger.error("Adding paused topics error", e);
        }
        logger.info("Topic [{}] was paused by command", tp.topic());
    } else {
        logger.info("Topic [{}] has been paused, the pause action was skipped", tp.topic());
    }
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
public static String committedFileName(String url, String topicsDir, String directory,
                                       TopicPartition topicPart, long startOffset, long endOffset,
                                       String extension, String zeroPadFormat) {
  String topic = topicPart.topic();
  int partition = topicPart.partition();
  StringBuilder sb = new StringBuilder();
  sb.append(topic);
  sb.append(HdfsSinkConnectorConstants.COMMMITTED_FILENAME_SEPARATOR);
  sb.append(partition);
  sb.append(HdfsSinkConnectorConstants.COMMMITTED_FILENAME_SEPARATOR);
  sb.append(String.format(zeroPadFormat, startOffset));
  sb.append(HdfsSinkConnectorConstants.COMMMITTED_FILENAME_SEPARATOR);
  sb.append(String.format(zeroPadFormat, endOffset));
  sb.append(extension);
  String name = sb.toString();
  return fileName(url, topicsDir, directory, name);
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
@Override
public Map<String, List<TopicPartition>> assign(Map<String, Integer> partitionsPerTopic,
                                                Map<String, Subscription> subscriptions) {
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    for (String memberId : subscriptions.keySet())
        assignment.put(memberId, new ArrayList<TopicPartition>());

    CircularIterator<String> assigner = new CircularIterator<>(Utils.sorted(subscriptions.keySet()));
    for (TopicPartition partition : allPartitionsSorted(partitionsPerTopic, subscriptions)) {
        final String topic = partition.topic();
        while (!subscriptions.get(assigner.peek()).topics().contains(topic))
            assigner.next();
        assignment.get(assigner.next()).add(partition);
    }
    return assignment;
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
/**
 * After each partition is parsed, we update the current metric totals with the total bytes
 * and number of records parsed. After all partitions have reported, we write the metric.
 */
public void record(TopicPartition partition, int bytes, int records) {
    this.unrecordedPartitions.remove(partition);
    this.fetchMetrics.increment(bytes, records);

    // collect and aggregate per-topic metrics
    String topic = partition.topic();
    FetchMetrics topicFetchMetric = this.topicFetchMetrics.get(topic);
    if (topicFetchMetric == null) {
        topicFetchMetric = new FetchMetrics();
        this.topicFetchMetrics.put(topic, topicFetchMetric);
    }
    topicFetchMetric.increment(bytes, records);

    if (this.unrecordedPartitions.isEmpty()) {
        // once all expected partitions from the fetch have reported in, record the metrics
        this.sensors.bytesFetched.record(topicFetchMetric.fetchBytes);
        this.sensors.recordsFetched.record(topicFetchMetric.fetchRecords);

        // also record per-topic metrics
        for (Map.Entry<String, FetchMetrics> entry: this.topicFetchMetrics.entrySet()) {
            FetchMetrics metric = entry.getValue();
            this.sensors.recordTopicFetchMetrics(entry.getKey(), metric.fetchBytes, metric.fetchRecords);
        }
    }
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
private void addToResetList(final TopicPartition partition, final Set<TopicPartition> partitions, final String logMessage, final String resetPolicy, final Set<String> loggedTopics) {
    final String topic = partition.topic();
    if (loggedTopics.add(topic)) {
        log.info(logMessage, logPrefix, topic, resetPolicy);
    }
    partitions.add(partition);
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
private ConsumerPair removeMovementRecordOfPartition(TopicPartition partition) {
    ConsumerPair pair = partitionMovements.remove(partition);

    String topic = partition.topic();
    Map<ConsumerPair, Set<TopicPartition>> partitionMovementsForThisTopic = partitionMovementsByTopic.get(topic);
    partitionMovementsForThisTopic.get(pair).remove(partition);
    if (partitionMovementsForThisTopic.get(pair).isEmpty())
        partitionMovementsForThisTopic.remove(pair);
    if (partitionMovementsByTopic.get(topic).isEmpty())
        partitionMovementsByTopic.remove(topic);

    return pair;
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
private void addPartitionMovementRecord(TopicPartition partition, ConsumerPair pair) {
    partitionMovements.put(partition, pair);

    String topic = partition.topic();
    if (!partitionMovementsByTopic.containsKey(topic))
        partitionMovementsByTopic.put(topic, new HashMap<ConsumerPair, Set<TopicPartition>>());

    Map<ConsumerPair, Set<TopicPartition>> partitionMovementsForThisTopic = partitionMovementsByTopic.get(topic);
    if (!partitionMovementsForThisTopic.containsKey(pair))
        partitionMovementsForThisTopic.put(pair, new HashSet<TopicPartition>());

    partitionMovementsForThisTopic.get(pair).add(partition);
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
/**
 * Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment
 * and will replace the previous assignment (if there is one).
 *
 * If the given list of topic partitions is empty, it is treated the same as {@link #unsubscribe()}.
 *
 * <p>
 * Manual topic assignment through this method does not use the consumer's group management
 * functionality. As such, there will be no rebalance operation triggered when group membership or cluster and topic
 * metadata change. Note that it is not possible to use both manual partition assignment with {@link #assign(Collection)}
 * and group assignment with {@link #subscribe(Collection, ConsumerRebalanceListener)}.
 *
 * @param partitions The list of partitions to assign this consumer
 * @throws IllegalArgumentException If partitions is null or contains null or empty topics
 */
@Override
// 用户手动订阅指定的topic并指定消费的分区，和subscribe方法互斥
public void assign(Collection<TopicPartition> partitions) {
    acquire();
    try {
        if (partitions == null) {
            throw new IllegalArgumentException("Topic partition collection to assign to cannot be null");
        } else if (partitions.isEmpty()) {
            this.unsubscribe();
        } else {
            Set<String> topics = new HashSet<>();
            for (TopicPartition tp : partitions) {
                String topic = (tp != null) ? tp.topic() : null;
                if (topic == null || topic.trim().isEmpty())
                    throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic");
                topics.add(topic);
            }

            // make sure the offsets of topic partitions the consumer is unsubscribing from
            // are committed since there will be no following rebalance
            this.coordinator.maybeAutoCommitOffsetsNow();

            log.debug("Subscribed to partition(s): {}", Utils.join(partitions, ", "));
            this.subscriptions.assignFromUser(new HashSet<>(partitions));
            metadata.setTopics(topics);
        }
    } finally {
        release();
    }
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
/**
 * group partitions by topic
 * @param partitions
 * @return partitions per topic
 */
public static Map<String, List<Integer>> groupDataByTopic(List<TopicPartition> partitions) {
    Map<String, List<Integer>> partitionsByTopic = new HashMap<>();
    for (TopicPartition tp: partitions) {
        String topic = tp.topic();
        List<Integer> topicData = partitionsByTopic.get(topic);
        if (topicData == null) {
            topicData = new ArrayList<>();
            partitionsByTopic.put(topic, topicData);
        }
        topicData.add(tp.partition());
    }
    return  partitionsByTopic;
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
public static String directoryName(String url, String topicsDir, TopicPartition topicPart) {
  String topic = topicPart.topic();
  int partition = topicPart.partition();
  return url + "/" + topicsDir + "/" + topic + "/" + partition;
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
public static String fileName(String url, String topicsDir, TopicPartition topicPart,
                              String name) {
  String topic = topicPart.topic();
  int partition = topicPart.partition();
  return url + "/" + topicsDir + "/" + topic + "/" + partition + "/" + name;
}
 

import org.apache.kafka.common.TopicPartition; //导入方法依赖的package包/类
public static List<OffsetInfo> getOffsetQuarz() {

        Map<String, Map<String, List<OffsetInfo>>> groupTopicPartitionListMap = new ConcurrentHashMap<>();

        for (Map.Entry<GroupTopicPartition, OffsetAndMetadata> entry: kafkaConsumerOffsets.entrySet()) {
            GroupTopicPartition groupTopicPartition = entry.getKey();
            OffsetAndMetadata offsetAndMetadata = entry.getValue();
            String group = groupTopicPartition.group();
            TopicPartition topicPartition = groupTopicPartition.topicPartition();
            String topic = topicPartition.topic();
            int partition = topicPartition.partition();
            Long committedOffset = offsetAndMetadata.offset();

            if (!logEndOffsetMap.containsKey(topicPartition)) {
                logger.error("The logEndOffsetMap not contains " + topicPartition);
                return null;
            }
            long logSize = logEndOffsetMap.get(topicPartition);

            // May the refresh operation thread take some time to update
            logSize = logSize >= committedOffset ? logSize : committedOffset;
            long lag = committedOffset == -1 ? 0 : (logSize - committedOffset);

            OffsetInfo offsetInfo = new OffsetInfo();
            offsetInfo.setGroup(group);
            offsetInfo.setTopic(topic);
            offsetInfo.setCommittedOffset(committedOffset);
            offsetInfo.setLogSize(logSize);
            offsetInfo.setLag(lag);
            offsetInfo.setTimestamp(offsetAndMetadata.commitTimestamp());

            if (!groupTopicPartitionListMap.containsKey(group)) {
                Map<String, List<OffsetInfo>> topicPartitionMap = new ConcurrentHashMap<>();
                groupTopicPartitionListMap.put(group, topicPartitionMap);
            }
            if (!groupTopicPartitionListMap.get(group).containsKey(topic)) {
                List<OffsetInfo> offsetInfos = new ArrayList<>();
                groupTopicPartitionListMap.get(group).put(topic, offsetInfos);
            }
            groupTopicPartitionListMap.get(group).get(topic).add(offsetInfo);

        }
        return flattenNestedMap(groupTopicPartitionListMap);
    }