Java KafkaServer类代码示例

import kafka.server.KafkaServer; //导入依赖的package包/类
public void initialize() {
    if (initialized) {
        throw new IllegalStateException("Context has been already initialized");
    }
    zkServer = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect());
    zkClient = new ZkClient(zkServer.connectString(), 10000, 10000, ZKStringSerializer$.MODULE$);

    port = TestUtils.choosePort();

    KafkaConfig config = new KafkaConfig(TestUtils.createBrokerConfig(brokerId, port, true));
    Time mock = new MockTime();

    kafkaServer = new KafkaServer(config, mock);
    kafkaServer.startup();

    initialized = true;
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
@Test
public void underreplicatedTopicsCanBeCreatedAndVerified() {
  // Given
  KafkaUtilities kUtil = new KafkaUtilities();
  EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster();
  int numBrokers = 1;
  int partitions = numBrokers + 1;
  int replication = numBrokers + 1;
  cluster.startCluster(numBrokers);
  KafkaServer broker = cluster.getBroker(0);
  KafkaZkClient zkClient = broker.zkClient();

  // When/Then
  for (String topic : exampleTopics) {
    assertThat(kUtil.createAndVerifyTopic(zkClient, topic, partitions, replication, oneYearRetention)).isTrue();
    // Only one broker is up, so the actual number of replicas will be only 1.
    assertThat(kUtil.verifySupportTopic(zkClient, topic, partitions, replication)).isEqualTo(KafkaUtilities.VerifyTopicState.Less);
  }
  assertThat(kUtil.getNumTopics(zkClient)).isEqualTo(exampleTopics.length);

  // Cleanup
  cluster.stopCluster();
  zkClient.close();
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public void startup() {
    for (int i = 0; i < ports.size(); i++) {
        Integer port = ports.get(i);
        File logDir = TestUtils.constructTempDir("kafka-local");

        Properties properties = new Properties();
        properties.putAll(baseProperties);
        properties.setProperty("zookeeper.connect", zkConnection);
        properties.setProperty("broker.id", String.valueOf(i + 1));
        properties.setProperty("host.name", "localhost");
        properties.setProperty("port", Integer.toString(port));
        properties.setProperty("log.dir", logDir.getAbsolutePath());
        properties.setProperty("log.flush.interval.messages", String.valueOf(1));

        KafkaServer broker = startBroker(properties);

        brokers.add(broker);
        logDirs.add(logDir);
    }
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public static void waitUntilMetadataIsPropagated(final List<KafkaServer> servers,
                                                 final String topic,
                                                 final int partition,
                                                 final long timeout) throws InterruptedException {
    TestUtils.waitForCondition(new TestCondition() {
        @Override
        public boolean conditionMet() {
            for (final KafkaServer server : servers) {
                final MetadataCache metadataCache = server.apis().metadataCache();
                final Option<PartitionStateInfo> partitionInfo =
                        metadataCache.getPartitionInfo(topic, partition);
                if (partitionInfo.isEmpty()) {
                    return false;
                }
                final PartitionStateInfo partitionStateInfo = partitionInfo.get();
                if (!Request.isValidBrokerId(partitionStateInfo.leaderIsrAndControllerEpoch().leaderAndIsr().leader())) {
                    return false;
                }
            }
            return true;
        }
    }, timeout, "metadata for topic=" + topic + " partition=" + partition + " not propagated to all brokers");

}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
  return new KafkaServer(kafkaConfig, new Time() {

    @Override
    public long milliseconds() {
      return System.currentTimeMillis();
    }

    @Override
    public long nanoseconds() {
      return System.nanoTime();
    }

    @Override
    public void sleep(long ms) {
      try {
        Thread.sleep(ms);
      } catch (InterruptedException e) {
        Thread.interrupted();
      }
    }
  });
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public void startup() {
    for (int i = 0; i < ports.size(); i++) {
        Integer port = ports.get(i);
        File logDir = TestUtils.constructTempDir("kafka-local");

        Properties properties = new Properties();
        properties.putAll(baseProperties);
        properties.setProperty("zookeeper.connect", zkConnection);
        properties.setProperty("broker.id", String.valueOf(i + 1));
        properties.setProperty("host.name", "localhost");
        properties.setProperty("port", Integer.toString(port));
        properties.setProperty("log.dir", logDir.getAbsolutePath());
        properties.setProperty("num.partitions", String.valueOf(1));
        properties.setProperty("auto.create.topics.enable", String.valueOf(Boolean.TRUE));
        System.out.println("EmbeddedKafkaCluster: local directory: " + logDir.getAbsolutePath());
        properties.setProperty("log.flush.interval.messages", String.valueOf(1));

        KafkaServer broker = startBroker(properties);

        brokers.add(broker);
        logDirs.add(logDir);
    }
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public static void waitUntilMetadataIsPropagated(final List<KafkaServer> servers,
                                                 final String topic,
                                                 final int partition,
                                                 final long timeout) throws InterruptedException {
  TestUtils.waitForCondition(new TestCondition() {
    @Override
    public boolean conditionMet() {
      for (final KafkaServer server : servers) {
        final MetadataCache metadataCache = server.apis().metadataCache();
        final Option<UpdateMetadataRequest.PartitionState> partitionInfo =
                metadataCache.getPartitionInfo(topic, partition);
        if (partitionInfo.isEmpty()) {
          return false;
        }
        final UpdateMetadataRequest.PartitionState metadataPartitionState = partitionInfo.get();
        if (!Request.isValidBrokerId(metadataPartitionState.basePartitionState.leader)) {
          return false;
        }
      }
      return true;
    }
  }, timeout, "metadata for topic=" + topic + " partition=" + partition + " not propagated to all brokers");

}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public void publish(String topic, List<String> messages)
{
  Properties producerProps = new Properties();
  producerProps.setProperty("bootstrap.servers", BROKERHOST + ":" + BROKERPORT);
  producerProps.setProperty("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
  producerProps.setProperty("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");

  try (KafkaProducer<Integer, byte[]> producer = new KafkaProducer<>(producerProps)) {
    for (String message : messages) {
      ProducerRecord<Integer, byte[]> data = new ProducerRecord<>(topic, message.getBytes(StandardCharsets.UTF_8));
      producer.send(data);
    }
  }

  List<KafkaServer> servers = new ArrayList<KafkaServer>();
  servers.add(kafkaServer);
  TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 30000);
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
public void startup() {
	for (int i = 0; i < ports.size(); i++) {
		Integer port = ports.get(i);
		File logDir = TestUtils.constructTempDir("kafka-local");

		Properties properties = new Properties();
		properties.putAll(baseProperties);
		properties.setProperty("zookeeper.connect", zkConnection);
		properties.setProperty("broker.id", String.valueOf(i + 1));
		properties.setProperty("host.name", "localhost");
		properties.setProperty("port", Integer.toString(port));
		properties.setProperty("log.dir", logDir.getAbsolutePath());
		properties.setProperty("num.partitions",  String.valueOf(1));
		properties.setProperty("auto.create.topics.enable",  String.valueOf(Boolean.TRUE));
		System.out.println("EmbeddedKafkaCluster: local directory: " + logDir.getAbsolutePath());
		properties.setProperty("log.flush.interval.messages", String.valueOf(1));

		KafkaServer broker = startBroker(properties);

		brokers.add(broker);
		logDirs.add(logDir);
	}
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
/**
 * @param server The Kafka server.
 * @param kafkaSupportConfig The properties this server was created from, plus extra Proactive
 *     Support (PS) ones
 *     Note that Kafka does not understand PS properties,
 *     hence server->KafkaConfig() does not contain any of them, necessitating
 *     passing this extra argument to the API.
 * @param serverRuntime The Java runtime of the server that is being monitored.
 * @param kafkaUtilities An instance of {@link KafkaUtilities} that will be used to perform
 *     e.g. Kafka topic management if needed.
 */
public MetricsReporter(
    KafkaServer server,
    KafkaSupportConfig kafkaSupportConfig,
    Runtime serverRuntime,
    KafkaUtilities kafkaUtilities
) {
  super(kafkaSupportConfig, kafkaUtilities, null, true);
  this.server = server;
  this.serverRuntime = serverRuntime;
  this.kafkaSupportConfig = kafkaSupportConfig;
  this.zkClientProvider = new KafkaServerZkClientProvider(server);
  Objects.requireNonNull(server, "Kafka Server can't be null");
  Objects.requireNonNull(serverRuntime, "serverRuntime can't be null");

}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
@Test
public void underreplicatedTopicsCanBeRecreatedAndVerified() {
  // Given
  KafkaUtilities kUtil = new KafkaUtilities();
  EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster();
  int numBrokers = 1;
  int partitions = numBrokers + 1;
  int replication = numBrokers + 1;
  cluster.startCluster(numBrokers);
  KafkaServer broker = cluster.getBroker(0);
  KafkaZkClient zkClient = broker.zkClient();

  // When/Then
  for (String topic : exampleTopics) {
    assertThat(kUtil.createAndVerifyTopic(zkClient, topic, partitions, replication, oneYearRetention)).isTrue();
    assertThat(kUtil.createAndVerifyTopic(zkClient, topic, partitions, replication, oneYearRetention)).isTrue();
    assertThat(kUtil.verifySupportTopic(zkClient, topic, partitions, replication)).isEqualTo(KafkaUtilities.VerifyTopicState.Less);
  }
  assertThat(kUtil.getNumTopics(zkClient)).isEqualTo(exampleTopics.length);

  // Cleanup
  cluster.stopCluster();
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
@Test
public void replicatedTopicsCanBeCreatedAndVerified() {
  // Given
  KafkaUtilities kUtil = new KafkaUtilities();
  EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster();
  int numBrokers = 3;
  cluster.startCluster(numBrokers);
  KafkaServer broker = cluster.getBroker(0);
  KafkaZkClient zkClient = broker.zkClient();
  Random random = new Random();
  int replication = numBrokers;

  // When/Then
  for (String topic : exampleTopics) {
    int morePartitionsThanBrokers = random.nextInt(10) + numBrokers + 1;
    assertThat(kUtil.createAndVerifyTopic(zkClient, topic, morePartitionsThanBrokers, replication, oneYearRetention)).isTrue();
    assertThat(kUtil.verifySupportTopic(zkClient, topic, morePartitionsThanBrokers, replication)).isEqualTo(KafkaUtilities.VerifyTopicState.Exactly);
  }

  // Cleanup
  cluster.stopCluster();
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
@AfterClass
public static void stop() {
    for (KafkaServer kafkaServer : kafkaServers) {
        kafkaServer.shutdown();
    }

    kafkaServers.clear();
    zkClient.close();
    zkServer.shutdown();

    Utils.delete(zkData);
    Utils.delete(zkLogs);
}
 

import kafka.server.KafkaServer; //导入依赖的package包/类
@Override
protected void before() throws Throwable {
    logDirectory = tempDir(perTest("kafka-log"));
    Properties properties = brokerDefinition.getProperties();
    properties.setProperty(KafkaConfig.LogDirProp(), logDirectory.getCanonicalPath());
    kafkaServer = new KafkaServer(new KafkaConfig(properties),
            SystemTime$.MODULE$, Some$.MODULE$.apply("kafkaServer"));
    kafkaServer.startup();

    List<TopicDefinition> topicDefinitions = brokerDefinition.getTopicDefinitions();
    if (!topicDefinitions.isEmpty()) {
        ZkUtils zkUtils = ZkUtils.apply(brokerDefinition.getZookeeperConnect(), 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        for (TopicDefinition topicDefinition : topicDefinitions) {
            String name = topicDefinition.getName();
            log.info("Creating topic {}", name);
            AdminUtils.createTopic(zkUtils,
                    name,
                    topicDefinition.getPartitions(),
                    topicDefinition.getReplicationFactor(),
                    topicDefinition.getProperties());
        }
    }
}