Java KafkaProducer.flush方法代码示例

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
public void publishDummyData() {
    final String topic = "TestTopic";

    // Create publisher
    final Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

    final KafkaProducer<String, String> producer = new KafkaProducer<>(config);
    for (int charCode = 65; charCode < 91; charCode++) {
        final char[] key = new char[1];
        key[0] = (char) charCode;

        producer.send(new ProducerRecord<>(topic, new String(key), new String(key)));
    }
    producer.flush();
    producer.close();
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
public void publishDummyDataNumbers() {
    final String topic = "NumbersTopic";

    // Create publisher
    final Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

    final KafkaProducer<Integer, Integer> producer = new KafkaProducer<>(config);
    for (int value = 0; value < 10000; value++) {
        producer.send(new ProducerRecord<>(topic, value, value));
    }
    producer.flush();
    producer.close();
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
private void flushToTopic(final StatAggregator statAggregator,
                          final String topic,
                          final EventStoreTimeIntervalEnum newInterval,
                          final KafkaProducer<StatEventKey, StatAggregate> producer) {

    Preconditions.checkNotNull(statAggregator);
    Preconditions.checkNotNull(producer);

    //as our logger uses a lambda we need to assign this and make it final which is not ideal if
    //we are not in debug mode
    final Instant startTime = Instant.now();

    //Uplift the statkey to the new aggregationInterval and put it on the topic
    //We will not be trying to uplift the statKey if we are already at the highest aggregationInterval
    //so the RTE that cloneAndChangeInterval can throw should never happen
    statAggregator.getAggregates().entrySet().stream()
            .map(entry -> new ProducerRecord<>(
                    topic,
                    entry.getKey().cloneAndChangeInterval(newInterval),
                    entry.getValue()))
            .peek(producerRecord -> LOGGER.trace("Putting record {} on topic {}", producerRecord, topic))
            .forEach(producer::send);

    LOGGER.debug(() -> String.format("Flushed %s records from interval %s with new interval %s to topic %s in %sms",
            statAggregator.size(), statAggregator.getAggregationInterval(), newInterval, topic, Duration.between(startTime, Instant.now()).toMillis()));

    producer.flush();
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
@Test
public void testOffsetsNotCommittedOnStop() throws Exception {
  String message = "testOffsetsNotCommittedOnStop-" + System.nanoTime();

  KafkaChannel channel = startChannel(false);

  KafkaProducer<String, byte[]> producer =
      new KafkaProducer<String, byte[]>(channel.getProducerProps());
  ProducerRecord<String, byte[]> data =
      new ProducerRecord<String, byte[]>(topic, "header-" + message, message.getBytes());
  producer.send(data).get();
  producer.flush();
  producer.close();

  Event event = takeEventWithoutCommittingTxn(channel);
  Assert.assertNotNull(event);
  Assert.assertTrue(Arrays.equals(message.getBytes(), event.getBody()));

  // Stop the channel without committing the transaction
  channel.stop();

  channel = startChannel(false);

  // Message should still be available
  event = takeEventWithoutCommittingTxn(channel);
  Assert.assertNotNull(event);
  Assert.assertTrue(Arrays.equals(message.getBytes(), event.getBody()));
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
public void testProduce() throws Exception {
    Properties producerProps = new Properties();
    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, testConfig.bootstrapServer);
    ByteArraySerializer serializer = new ByteArraySerializer();
    KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps, serializer, serializer);
    ProducerRecord<byte[], byte[]> record1 = new ProducerRecord<>(testConfig.topic, message1);
    Future<RecordMetadata> future1 = producer.send(record1);
    ProducerRecord<byte[], byte[]> record2 = new ProducerRecord<>(testConfig.topic, message2);
    Future<RecordMetadata> future2 = producer.send(record2);
    producer.flush();
    future1.get();
    future2.get();
    producer.close();
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {
    // set up the producer
    KafkaProducer<String, String> producer;
    try (InputStream props = Resources.getResource("producer.props").openStream()) {
        Properties properties = new Properties();
        properties.load(props);
        producer = new KafkaProducer<>(properties);
    }

    try {
          int i =0;
          File file = new File("/home/leiming/DataFlow/imply-2.2.3/quickstart/wikiticker-2016-06-27-sampled.json");
          BufferedReader br = new BufferedReader(new FileReader(file));
          String st;
          while((st=br.readLine())!=null) {
              // send lots of messages
              /**
              producer.send(new ProducerRecord<String, String>(
                      "fast-messages",
                      String.format("{\"type\":\"test\", \"t\":%.3f, \"k\":%d}", System.nanoTime() * 1e-9, i)));
                        **/
              // every so often send to a different topic

                  producer.send(new ProducerRecord<String, String>(
                          "leidaxia",
                       //   String.format("{\"type\":\"marker\", \"t\":%.3f, \"k\":%d}", System.nanoTime() * 1e-9, i)));
                   st));
                 /**producer.send(new ProducerRecord<String, String>(
                          "summary-markers",
                          String.format("{\"type\":\"other\", \"t\":%.3f, \"k\":%d}", System.nanoTime() * 1e-9, i)));
                **/
                  producer.flush();
                  System.out.println("Sent msg num" + i);
                  System.out.println("Sent msg " + st);
                  i =i + 1;
              }

    } catch (Throwable throwable) {
        System.out.printf("%s", throwable.getStackTrace());
    } finally {
        producer.close();
    }

}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
/**
 * Produce some records into the defined kafka namespace.
 *
 * @param keysAndValues Records you want to produce.
 * @param topicName the namespace name to produce into.
 * @param partitionId the partition to produce into.
 * @return List of ProducedKafkaRecords.
 */
public List<ProducedKafkaRecord<byte[], byte[]>> produceRecords(
    final Map<byte[], byte[]> keysAndValues,
    final String topicName,
    final int partitionId
) {
    // This holds the records we produced
    List<ProducerRecord<byte[], byte[]>> producedRecords = Lists.newArrayList();

    // This holds futures returned
    List<Future<RecordMetadata>> producerFutures = Lists.newArrayList();

    KafkaProducer<byte[], byte[]> producer = kafkaTestServer.getKafkaProducer(
        ByteArraySerializer.class,
        ByteArraySerializer.class
    );
    for (Map.Entry<byte[], byte[]> entry: keysAndValues.entrySet()) {
        // Construct filter
        ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topicName, partitionId, entry.getKey(), entry.getValue());
        producedRecords.add(record);

        // Send it.
        producerFutures.add(producer.send(record));
    }

    // Publish to the namespace and close.
    producer.flush();
    logger.info("Produce completed");
    producer.close();

    // Loop thru the futures, and build KafkaRecord objects
    List<ProducedKafkaRecord<byte[], byte[]>> kafkaRecords = Lists.newArrayList();
    try {
        for (int x = 0; x < keysAndValues.size(); x++) {
            final RecordMetadata metadata = producerFutures.get(x).get();
            final ProducerRecord<byte[], byte[]> producerRecord = producedRecords.get(x);

            kafkaRecords.add(ProducedKafkaRecord.newInstance(metadata, producerRecord));
        }
    } catch (InterruptedException | ExecutionException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }

    return kafkaRecords;
}
 

import org.apache.kafka.clients.producer.KafkaProducer; //导入方法依赖的package包/类
/**
 * Test that KafkaServer works as expected!
 *
 * This also serves as a decent example of how to use the producer and consumer.
 */
@Test
public void testProducerAndConsumer() throws Exception {
    final int partitionId = 0;

    // Define our message
    final String expectedKey = "my-key";
    final String expectedValue = "my test message";

    // Define the record we want to produce
    ProducerRecord<String, String> producerRecord = new ProducerRecord<>(topicName, partitionId, expectedKey, expectedValue);

    // Create a new producer
    KafkaProducer<String, String> producer = getKafkaTestServer().getKafkaProducer(StringSerializer.class, StringSerializer.class);

    // Produce it & wait for it to complete.
    Future<RecordMetadata> future = producer.send(producerRecord);
    producer.flush();
    while (!future.isDone()) {
        Thread.sleep(500L);
    }
    logger.info("Produce completed");

    // Close producer!
    producer.close();

    KafkaConsumer<String, String> kafkaConsumer =
        getKafkaTestServer().getKafkaConsumer(StringDeserializer.class, StringDeserializer.class);

    final List<TopicPartition> topicPartitionList = Lists.newArrayList();
    for (final PartitionInfo partitionInfo: kafkaConsumer.partitionsFor(topicName)) {
        topicPartitionList.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
    }
    kafkaConsumer.assign(topicPartitionList);
    kafkaConsumer.seekToBeginning(topicPartitionList);

    // Pull records from kafka, keep polling until we get nothing back
    ConsumerRecords<String, String> records;
    do {
        records = kafkaConsumer.poll(2000L);
        logger.info("Found {} records in kafka", records.count());
        for (ConsumerRecord<String, String> record: records) {
            // Validate
            assertEquals("Key matches expected", expectedKey, record.key());
            assertEquals("value matches expected", expectedValue, record.value());
        }
    }
    while (!records.isEmpty());

    // close consumer
    kafkaConsumer.close();
}