Java AdminUtils.createTopic方法代码示例

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create a topic
 *
 * @param connection        Connection
 * @param topicName         Topic name
 * @param partitions        The number of partitions for the topic being created
 * @param replicationFactor The replication factor for each partition in the topic being created
 * @param topicProperties   A topic configuration override for an existing topic
 * @throws TopicOperationException if topic was not created.
 */
public void createTopic(final ZkUtils connection, final String topicName,
                        final int partitions,
                        final int replicationFactor,
                        final Properties topicProperties) {

    try {
        AdminUtils.createTopic(connection,
                topicName,
                partitions,
                replicationFactor,
                topicProperties);

    } catch (IllegalArgumentException | KafkaException | AdminOperationException e) {
        throw new TopicOperationException(topicName, e.getMessage(), e, this.getClass());
    }
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
private void ensureTopicCreated(Map<String, ?> config) {
  ZkUtils zkUtils = createZkUtils(config);
  Map<String, List<PartitionInfo>> topics = _consumers.get(0).listTopics();
  long snapshotWindowMs = Long.parseLong((String) config.get(KafkaCruiseControlConfig.LOAD_SNAPSHOT_WINDOW_MS_CONFIG));
  int numSnapshotWindows = Integer.parseInt((String) config.get(KafkaCruiseControlConfig.NUM_LOAD_SNAPSHOTS_CONFIG));
  long retentionMs = (numSnapshotWindows * ADDITIONAL_SNAPSHOT_WINDOW_TO_RETAIN_FACTOR) * snapshotWindowMs;
  Properties props = new Properties();
  props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs));
  props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
  int replicationFactor = Math.min(2, zkUtils.getAllBrokersInCluster().size());
  if (!topics.containsKey(_partitionMetricSampleStoreTopic)) {
    AdminUtils.createTopic(zkUtils, _partitionMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
  } else {
    AdminUtils.changeTopicConfig(zkUtils, _partitionMetricSampleStoreTopic, props);
  }

  if (!topics.containsKey(_brokerMetricSampleStoreTopic)) {
    AdminUtils.createTopic(zkUtils, _brokerMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
  } else {
    AdminUtils.changeTopicConfig(zkUtils, _brokerMetricSampleStoreTopic, props);
  }

  KafkaCruiseControlUtils.closeZkUtilsWithTimeout(zkUtils, 10000);
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(final String topic,
                        final int partitions,
                        final int replication,
                        final Properties topicConfig) {
    log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
        topic, partitions, replication, topicConfig);

    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    final ZkClient zkClient = new ZkClient(
        zookeeperConnect(),
        DEFAULT_ZK_SESSION_TIMEOUT_MS,
        DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
        ZKStringSerializer$.MODULE$);
    final boolean isSecure = false;
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
    AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
    zkClient.close();
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // createTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  boolean isSecure = false;
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Creates a Topic.
 *
 * @param topicName             Topic name.
 * @param partitions        Number of partitions for the topic.
 * @param replicationFactor Replication factor.
 * @param curatorFramework CuratorFramework.
 */
public static void createTopic(String topicName, int partitions, int replicationFactor, CuratorFramework curatorFramework) {
    if (partitions <= 0)
        throw new AdminOperationException("number of partitions must be larger than 0");

    if (replicationFactor <= 0)
        throw new AdminOperationException("replication factor must be larger than 0");

    if (!topicExists(topicName, curatorFramework)) {
        m_logger.info(String.format("Topic %s not found, creating...", topicName));
        ZkClient zkClient = fromCurator(curatorFramework);
        try {
            AdminUtils.createTopic(zkClient, topicName, partitions, replicationFactor, new Properties());
            m_logger.info("Topic created. name: {}, partitions: {}, replicationFactor: {}", topicName,
                    partitions, replicationFactor);
        } catch (TopicExistsException ignore) {
            m_logger.info("Topic exists. name: {}", topicName);
        } finally {
            if (zkClient != null) {
                zkClient.close();
            }
        }
    } else {
        m_logger.info(String.format("Topic %s found!", topicName));
    }
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(final String topic,
                        final int partitions,
                        final int replication,
                        final Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);

  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // createTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  final ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  final boolean isSecure = false;
  final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create the topic that the monitor uses to monitor the cluster.  This method attempts to create a topic so that all
 * the brokers in the cluster will have partitionToBrokerRatio partitions.  If the topic exists, but has different parameters
 * then this does nothing to update the parameters.
 *
 * TODO: Do we care about rack aware mode?  I would think no because we want to spread the topic over all brokers.
 * @param zkUrl zookeeper connection url
 * @param topic topic name
 * @param replicationFactor the replication factor for the topic
 * @param partitionToBrokerRatio This is multiplied by the number brokers to compute the number of partitions in the topic.
 * @param topicConfig additional parameters for the topic for example min.insync.replicas
 * @return the number of partitions created
 */
public static int createMonitoringTopicIfNotExists(String zkUrl, String topic, int replicationFactor,
    double partitionToBrokerRatio, Properties topicConfig) {
  ZkUtils zkUtils = ZkUtils.apply(zkUrl, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled());
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      return getPartitionNumForTopic(zkUrl, topic);
    }
    int brokerCount = zkUtils.getAllBrokersInCluster().size();
    int partitionCount = (int) Math.ceil(brokerCount * partitionToBrokerRatio);

    try {
      AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, topicConfig, RackAwareMode.Enforced$.MODULE$);
    } catch (TopicExistsException e) {
      //There is a race condition with the consumer.
      LOG.debug("Monitoring topic " + topic + " already exists in cluster " + zkUrl, e);
      return getPartitionNumForTopic(zkUrl, topic);
    }
    LOG.info("Created monitoring topic " + topic + " in cluster " + zkUrl + " with " + partitionCount + " partitions, min ISR of "
      + topicConfig.get(KafkaConfig.MinInSyncReplicasProp()) + " and replication factor of " + replicationFactor + ".");

    return partitionCount;
  } finally {
    zkUtils.close();
  }
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
@BeforeClass
public static void initKafka() throws Exception {
  synchronized (TestKafkaSuit.class) {
    if (initCount.get() == 0) {
      ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
      System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME).getFile());
      embeddedKafkaCluster = new EmbeddedKafkaCluster();
      Properties topicProps = new Properties();
      zkClient = new ZkClient(embeddedKafkaCluster.getZkServer().getConnectionString(), SESSION_TIMEOUT, CONN_TIMEOUT, ZKStringSerializer$.MODULE$);
      ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false);
      AdminUtils.createTopic(zkUtils, QueryConstants.JSON_TOPIC, 1, 1, topicProps, RackAwareMode.Disabled$.MODULE$);

      org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils
          .fetchTopicMetadataFromZk(QueryConstants.JSON_TOPIC, zkUtils);
      logger.info("Topic Metadata: " + fetchTopicMetadataFromZk);

      KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(),
          StringSerializer.class);
      generator.populateJsonMsgIntoKafka(QueryConstants.JSON_TOPIC, NUM_JSON_MSG);
    }
    initCount.incrementAndGet();
    runningSuite = true;
  }
  logger.info("Initialized Embedded Zookeeper and Kafka");
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // registerTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  boolean isSecure = false;
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
@Override
protected void before() throws Throwable {
    logDirectory = tempDir(perTest("kafka-log"));
    Properties properties = brokerDefinition.getProperties();
    properties.setProperty(KafkaConfig.LogDirProp(), logDirectory.getCanonicalPath());
    kafkaServer = new KafkaServer(new KafkaConfig(properties),
            SystemTime$.MODULE$, Some$.MODULE$.apply("kafkaServer"));
    kafkaServer.startup();

    List<TopicDefinition> topicDefinitions = brokerDefinition.getTopicDefinitions();
    if (!topicDefinitions.isEmpty()) {
        ZkUtils zkUtils = ZkUtils.apply(brokerDefinition.getZookeeperConnect(), 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        for (TopicDefinition topicDefinition : topicDefinitions) {
            String name = topicDefinition.getName();
            log.info("Creating topic {}", name);
            AdminUtils.createTopic(zkUtils,
                    name,
                    topicDefinition.getPartitions(),
                    topicDefinition.getReplicationFactor(),
                    topicDefinition.getProperties());
        }
    }
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
public static void createTopic(AppContext context,
                               String topicName,
                               int partitions,
                               int replicationFactor,
                               Schema schema) {

  // Currently the schema parameter is unused, but we have it here so it can
  // be applied to a schema registry for the topic when that is available.

  ZkClient client = new ZkClient(getKafkaProp(context, ZOOKEEPER_CONNECT), 1000, 1000, ZKStringSerializer$.MODULE$);

  try {
    if (!AdminUtils.topicExists(client, topicName)) {
      AdminUtils.createTopic(client, topicName, partitions, replicationFactor, new Properties());
    }
  } finally {
    client.close();
  }
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
public void createTopic(String topic) {
    log.debug("Create Topic {}.", topic);
    if (doesTopicExist(topic)) {
        log.debug("Topic %s already exists.", topic);
        return;
    }
     // Create a ZooKeeper client
    int sessionTimeoutMs = 10000;
    int connectionTimeoutMs = 10000;
    ZkClient zkClient = new ZkClient("localhost:2181", sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);

    // Create the topic
    int numPartitions = 1;
    int replicationFactor = 1;
    Properties topicConfig = new Properties();
    AdminUtils.createTopic(zkClient, topic, numPartitions, replicationFactor, topicConfig);
    log.debug("Topic {} is created.", topic);
}
 

import kafka.admin.AdminUtils; //导入方法依赖的package包/类
private void setupKafkaBroker() {
    try {
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", "/tmp/tmp_kafka_dir");
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "1500");
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();

        // create "sensordata" topic
        ZkClient zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
        AdminUtils.createTopic(zkClient, "sensordata", 1, 1, new Properties());
        zkClient.close();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}