Java ProducerConfiguration.setBatchingMaxMessages方法代码示例

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
private Set<String> publishMessagesBase(String topic, int count, boolean batching) throws Exception {
    Set<String> keys = new HashSet<>();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMaxPendingMessages(count);
    producerConf.setBatchingEnabled(batching);
    producerConf.setBatchingMaxMessages(BATCH_MAX_MESSAGES);
    producerConf.setBatchingMaxPublishDelay(Long.MAX_VALUE, TimeUnit.DAYS);

    try (Producer producer = pulsarClient.createProducer(topic, producerConf)) {
        Future<?> lastFuture = null;
        for (int i = 0; i < count; i++) {
            String key = "key"+i;
            byte[] data = ("my-message-" + i).getBytes();
            lastFuture = producer.sendAsync(MessageBuilder.create()
                                            .setKey(key)
                                            .setContent(data).build());
            keys.add(key);
        }
        lastFuture.get();
    }
    return keys;
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
MessageProducer(URL url, final DestinationName dest, boolean batch) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    ClientConfiguration conf = new ClientConfiguration();
    conf.setStatsInterval(0, TimeUnit.SECONDS);
    client = PulsarClient.create(url.toString(), conf);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    if (batch) {
        producerConfiguration.setBatchingEnabled(true);
        producerConfiguration.setBatchingMaxPublishDelay(1, TimeUnit.SECONDS);
        producerConfiguration.setBatchingMaxMessages(5);
    }
    producer = client.createProducer(topicName, producerConfiguration);

}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
public void testBatchMessagesRateOut() throws PulsarClientException, InterruptedException, PulsarAdminException {
    log.info("-- Starting {} test --", methodName);
    String topicName = "persistent://my-property/cluster/my-ns/testBatchMessagesRateOut";
    double produceRate = 17;
    int batchSize = 5;
    ConsumerConfiguration consumerConf = new ConsumerConfiguration();
    consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", consumerConf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxMessages(batchSize);
    producerConf.setBatchingEnabled(true);
    producerConf.setBatchingMaxPublishDelay(2, TimeUnit.SECONDS);

    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    AtomicBoolean runTest = new AtomicBoolean(true);
    Thread t1 = new Thread(() -> {
        RateLimiter r = RateLimiter.create(produceRate);
        while (runTest.get()) {
            r.acquire();
            producer.sendAsync("Hello World".getBytes());
            consumer.receiveAsync().thenAccept(message -> consumer.acknowledgeAsync(message));
        }
    });
    t1.start();
    Thread.sleep(2000); // Two seconds sleep
    runTest.set(false);
    pulsar.getBrokerService().updateRates();
    double actualRate = admin.persistentTopics().getStats(topicName).msgRateOut;
    assertTrue(actualRate > (produceRate / batchSize));
    consumer.unsubscribe();
    log.info("-- Exiting {} test --", methodName);
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic", "my-subscriber-name",
            conf);

    ProducerConfiguration producerConf = new ProducerConfiguration();

    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }

    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {

    int batchMessageDelayMs = 100;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(0);
    Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns-abc/topic1", "my-subscriber-name",
            conf);

    ProducerConfiguration producerConf = new ProducerConfiguration();

    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }

    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns-abc/topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime";
    final String subscriptionName = "time-size-sub-1" + compressionType.toString();

    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();

    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(5);
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);

    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);

    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs,
            topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);

    producer.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
private ProducerConfiguration getProducerConfiguration() {
    ProducerConfiguration conf = new ProducerConfiguration();

    // Set to false to prevent the server thread from being blocked if a lot of messages are pending.
    conf.setBlockIfQueueFull(false);

    if (queryParams.containsKey("sendTimeoutMillis")) {
        conf.setSendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("batchingEnabled")) {
        conf.setBatchingEnabled(Boolean.parseBoolean(queryParams.get("batchingEnabled")));
    }

    if (queryParams.containsKey("batchingMaxMessages")) {
        conf.setBatchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages")));
    }

    if (queryParams.containsKey("maxPendingMessages")) {
        conf.setMaxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages")));
    }

    if (queryParams.containsKey("batchingMaxPublishDelay")) {
        conf.setBatchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")),
                TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("messageRoutingMode")) {
        checkArgument(
                Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(),
                "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode"));
        MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode"));
        if (!MessageRoutingMode.CustomPartition.equals(routingMode)) {
            conf.setMessageRoutingMode(routingMode);
        }
    }

    if (queryParams.containsKey("compressionType")) {
        checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(),
                "Invalid compressionType %s", queryParams.get("compressionType"));
        conf.setCompressionType(CompressionType.valueOf(queryParams.get("compressionType")));
    }

    return conf;
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "batch_with_timeout")
public void testSyncProducerAndConsumer(int batchMessageDelayMs, int ackTimeoutSec) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    // Cumulative Ack-counter works if ackTimeOutTimer-task is enabled
    boolean isAckTimeoutTaskEnabledForCumulativeAck = ackTimeoutSec > 0;
    if (ackTimeoutSec > 0) {
        conf.setAckTimeout(ackTimeoutSec, TimeUnit.SECONDS);
    }

    Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic1", "my-subscriber-name",
            conf);

    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }

    Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic1", producerConf);

    int numMessages = 11;
    for (int i = 0; i < numMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < numMessages; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    Thread.sleep(2000);
    consumer.close();
    producer.close();
    validatingLogInfo(consumer, producer, isAckTimeoutTaskEnabledForCumulativeAck);
    log.info("-- Exiting {} test --", methodName);
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "batch")
public void testSendTimeout(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    ConsumerConfiguration consumerConf = new ConsumerConfiguration();
    consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/tp1/my-ns/my-topic5", "my-subscriber-name",
            consumerConf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(2 * batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    producerConf.setSendTimeout(1, TimeUnit.SECONDS);

    Producer producer = pulsarClient.createProducer("persistent://my-property/tp1/my-ns/my-topic5", producerConf);
    final String message = "my-message";

    // Trigger the send timeout
    stopBroker();

    Future<MessageId> future = producer.sendAsync(message.getBytes());

    try {
        future.get();
        fail("Send operation should have failed");
    } catch (ExecutionException e) {
        // Expected
    }

    startBroker();

    // We should not have received any message
    Message msg = consumer.receive(3, TimeUnit.SECONDS);
    assertNull(msg);
    consumer.close();
    producer.close();
    Thread.sleep(1000);
    ConsumerStats cStat = consumer.getStats();
    ProducerStats pStat = producer.getStats();
    assertEquals(pStat.getTotalMsgsSent(), 0);
    assertEquals(pStat.getTotalSendFailed(), 1);
    assertEquals(cStat.getTotalMsgsReceived(), 0);
    assertEquals(cStat.getTotalMsgsReceived(), cStat.getTotalAcksSent());
    log.info("-- Exiting {} test --", methodName);
}