Java ProducerRecord类代码示例

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Ignore
@Test
public void testWithString() throws Exception {
    KafkaComponent kafka = createComponent();
    
    Mono<List<String>> receive = Flux.from(kafka.from(TOPIC2, String.class))
        .take(2)
        .collectList();
    
    Subscriber<ProducerRecord> toTopic = kafka.to(TOPIC2, ProducerRecord.class);
    Flux.just(new ProducerRecord<String, String>(TOPIC2, "1", "test"),
              new ProducerRecord<String, String>(TOPIC2, "1", "test2"))
        .subscribe(toTopic);

    List<String> received = receive.block(Duration.ofSeconds(10));
    Assert.assertEquals(2, received.size());
    Assert.assertEquals("test", received.get(0));
    Assert.assertEquals("test2", received.get(1));
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Ignore
@Test
public void testWithConsumerRecord() throws Exception {
    KafkaComponent kafka = createComponent();
    
    Mono<List<Object>> receive = Flux.from(kafka.from(TOPIC1, ConsumerRecord.class))
        .map(record -> record.value())
        .take(2)
        .collectList();

    Subscriber<ProducerRecord> toTopic = kafka.to(TOPIC1, ProducerRecord.class);
    Flux.just(new ProducerRecord<String, String>(TOPIC1, "1", "test"),
              new ProducerRecord<String, String>(TOPIC1, "1", "test2"))
        .subscribe(toTopic);

    List<Object> received = receive.block(Duration.ofSeconds(10));
    Assert.assertEquals(2, received.size());
    Assert.assertEquals("test", received.get(0));
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Override
public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
  /*
  // Create wrappedRecord because headers can be read only in record (if record is sent second time)
  ProducerRecord<K, V> wrappedRecord = new ProducerRecord<>(record.topic(),
      record.partition(),
      record.timestamp(),
      record.key(),
      record.value(),
      record.headers());
  */

  try (Scope scope = buildAndInjectSpan(record)) {
    Callback wrappedCallback = new TracingCallback(callback, scope);
    return producer.send(record, wrappedCallback);
  }
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public void run() {
    int messageNo = 1;
    while (true) {
        String messageStr = "Message_" + messageNo;
        long startTime = System.currentTimeMillis();
        if (isAsync) { // Send asynchronously
            producer.send(new ProducerRecord<>(topic,
                messageNo,
                messageStr), new DemoCallBack(startTime, messageNo, messageStr));
        } else { // Send synchronously
            try {
                producer.send(new ProducerRecord<>(topic,
                    messageNo,
                    messageStr)).get();
                System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
        }
        ++messageNo;
    }
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Override
public <K, V, E> boolean send(Producer<K, V> producer, ProducerRecord<K, V> record, final E event,
                              final FailedDeliveryCallback<E> failedDeliveryCallback) {
    try {
        producer.send(record, new Callback() {
            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
                if (exception != null) {
                    failedDeliveryCallback.onFailedDelivery(event, exception);
                }
            }
        });
        return true;
    } catch (BufferExhaustedException e) {
        failedDeliveryCallback.onFailedDelivery(event, e);
        return false;
    }
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public void start() throws InterruptedException {
  RandomGenerator random = RandomManager.getRandom();

  Properties props = ConfigUtils.keyValueToProperties(
      "bootstrap.servers", "localhost:" + kafkaPort,
      "key.serializer", "org.apache.kafka.common.serialization.StringSerializer",
      "value.serializer", "org.apache.kafka.common.serialization.StringSerializer",
      "compression.type", "gzip",
      "batch.size", 0,
      "acks", 1,
      "max.request.size", 1 << 26 // TODO
  );
  try (Producer<String,String> producer = new KafkaProducer<>(props)) {
    for (int i = 0; i < howMany; i++) {
      Pair<String,String> datum = datumGenerator.generate(i, random);
      ProducerRecord<String,String> record =
          new ProducerRecord<>(topic, datum.getFirst(), datum.getSecond());
      producer.send(record);
      log.debug("Sent datum {} = {}", record.key(), record.value());
      if (intervalMsec > 0) {
        Thread.sleep(intervalMsec);
      }
    }
  }
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
    switch (method.getName()) {
        case SEND_METHOD_NAME: {
            ProducerRecord record = (ProducerRecord) args[0];

            args[0] = new ProducerRecord<>(
                    BaseIntegrationTest.adjustTopicNameForTest(record.topic()),
                    record.partition(),
                    record.timestamp(),
                    record.key(),
                    record.value()
            );
            break;
        }
        case PARTITIONS_FOR_METHOD_NAME: {
            args[0] = BaseIntegrationTest.adjustTopicNameForTest((String) args[0]);
            break;
        }
    }
    return method.invoke(producer, args);
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public static void main(String[] args) {

        Map<String, Object> config = new HashMap<String, Object>();
        config.put("partitioner.class", "com.wngn.kafka.SimpleKeyPartition");
        LatestProducer producer = LatestProducer.getInstance(ProducerConstants.TOPIC_KAFKA_TEST, config);
        ProducerRecord<String, String> record = null;
        long index = 0L;
        boolean controller = true;
        while (controller) {
            controller = false;
            index++;
            System.out.println(index + "------------");
            try {
                String message = "message_" + index;
                RecordMetadata recordMetadata = producer.sendWithSync("1", message);
                System.out.format("PARTITION: %d OFFSET: %d\n", recordMetadata.partition(), recordMetadata.offset());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        producer.close();
    }
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Test
public void shouldNotBeLoggingEnabledStoreWhenLoggingNotEnabled() throws Exception {
    store = createStore(false, false);
    final List<ProducerRecord> logged = new ArrayList<>();
    final NoOpRecordCollector collector = new NoOpRecordCollector() {
        @Override
        public <K, V> void send(final String topic,
                                K key,
                                V value,
                                Integer partition,
                                Long timestamp,
                                Serializer<K> keySerializer,
                                Serializer<V> valueSerializer) {
            logged.add(new ProducerRecord<K, V>(topic, partition, timestamp, key, value));
        }
    };
    final MockProcessorContext context = new MockProcessorContext(TestUtils.tempDirectory(),
                                                                  Serdes.String(),
                                                                  Serdes.String(),
                                                                  collector,
                                                                  cache);
    context.setTime(1);
    store.init(context, store);
    store.put("a", "b");
    assertTrue(logged.isEmpty());
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public ConsumerRecords<K, V> nextBatch() {
    List<Future<RecordMetadata>> markerSends = new ArrayList<>();

    // 1. Get messages from topic, in batches
    ConsumerRecords<K, V> records = msgConsumer.poll(msgPollTimeout);
    for (ConsumerRecord<K, V> record : records) {
        // 2. Write a "start" marker. Collecting the future responses.
        markerSends.add(markerProducer.send(
                new ProducerRecord<>(config.getMarkerTopic(),
                        MarkerKey.fromRecord(record),
                        new StartMarker(config.getMsgTimeoutMs()))));
    }

    // Waiting for a confirmation that each start marker has been sent
    markerSends.forEach(f -> {
        try { f.get(); } catch (Exception e) { throw new RuntimeException(e); }
    });

    // 3. after all start markers are sent, commit offsets. This needs to be done as close to writing the
    // start marker as possible, to minimize the number of double re-processed messages in case of failure.
    msgConsumer.commitSync();

    return records;
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
private static void produceRecords(String bootstrapServers) {
    Properties properties = new Properties();
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName());
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());

    Producer<Long, byte[]> producer = new KafkaProducer<>(properties);

    LongStream.rangeClosed(1, 100).boxed()
            .map(number ->
                    new ProducerRecord<>(
                            TOPIC, //topic
                            number, //key
                            String.format("record-%s", number.toString()).getBytes())) //value
            .forEach(record -> producer.send(record));
    producer.close();
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public void publish(BrokerStats brokerStats) throws IOException {
  try {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream, null);

    avroEventWriter.write(brokerStats, binaryEncoder);
    binaryEncoder.flush();
    IOUtils.closeQuietly(stream);

    String key = brokerStats.getName() + "_" + System.currentTimeMillis();
    int numPartitions = kafkaProducer.partitionsFor(destTopic).size();
    int partition = brokerStats.getId() % numPartitions;

    Future<RecordMetadata> future = kafkaProducer.send(
        new ProducerRecord(destTopic, partition, key.getBytes(), stream.toByteArray()));
    future.get();

    OpenTsdbMetricConverter.incr("kafka.stats.collector.success", 1, "host=" + HOSTNAME);
  } catch (Exception e) {
    LOG.error("Failure in publish stats", e);
    OpenTsdbMetricConverter.incr("kafka.stats.collector.failure", 1, "host=" + HOSTNAME);
    throw new RuntimeException("Avro serialization failure", e);
  }
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
public static void  sendStringMessage() throws Exception{
	Properties props = new Properties();
	props.put("bootstrap.servers", servers);
	props.put("acks", "all");
	props.put("retries", 0);
	props.put("batch.size", 16384);
	props.put("linger.ms", 1);
	props.put("buffer.memory", 33554432);
	props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
	props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

	Producer<String, String> producer = new org.apache.kafka.clients.producer.KafkaProducer<>(props);

	//没有任何分区，默认1个分区，发送消息
	int i=0;
	while(i<1000){
		Thread.sleep(1000L);
		String message = "zhangsan"+i;
		producer.send(new ProducerRecord<>("NL_U_APP_ALARM_APP_STRING",message));
		i++;
		producer.flush();
	}
	producer.close();
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
private void checkMulticast(KafkaUsage usage, String topic, KafkaSource<Integer> source) {
  List<Integer> resultsA = new ArrayList<>();
  List<Integer> resultsB = new ArrayList<>();
  source
    .transformPayload(i -> i + 1)
    .to(Sink.forEachPayload(resultsB::add));

  source
    .transformPayload(i -> i + 1)
    .to(Sink.forEachPayload(resultsA::add));

  AtomicInteger counter = new AtomicInteger();
  usage.produceIntegers(10, null,
    () -> new ProducerRecord<>(topic, counter.getAndIncrement()));

  await().atMost(1, TimeUnit.MINUTES).until(() -> resultsA.size() >= 10);
  await().atMost(1, TimeUnit.MINUTES).until(() -> resultsB.size() >= 10);
  assertThat(resultsA).containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
  assertThat(resultsB).containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
}
 

import org.apache.kafka.clients.producer.ProducerRecord; //导入依赖的package包/类
@Test
public void testSource() throws InterruptedException {
  KafkaUsage usage = new KafkaUsage();
  String topic = UUID.randomUUID().toString();
  List<Integer> results = new ArrayList<>();
  KafkaSource<Integer> source = new KafkaSource<>(vertx,
    getKafkaConfig()
      .put("topic", topic)
      .put("value.serializer", IntegerSerializer.class.getName())
      .put("value.deserializer", IntegerDeserializer.class.getName())
  );
  source
    .transformPayload(i -> i + 1)
    .to(Sink.forEachPayload(results::add));

  AtomicInteger counter = new AtomicInteger();
  usage.produceIntegers(10, null,
    () -> new ProducerRecord<>(topic, counter.getAndIncrement()));

  await().atMost(1, TimeUnit.MINUTES).until(() -> results.size() >= 10);
  assertThat(results).containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
}