Java ProducerConfiguration.setCompressionType方法代码示例

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.create(SERVICE_URL);

    // Here you get the chance to configure producer specific settings. eg:
    ProducerConfiguration conf = new ProducerConfiguration();

    // Enable compression
    conf.setCompressionType(CompressionType.LZ4);

    // Once the producer is created, it can be used for the entire application life-cycle
    Producer producer = client.createProducer(TOPIC_NAME, conf);
    log.info("Created Pulsar producer");

    // Send few test messages
    for (int i = 0; i < 10; i++) {
        String content = String.format("hello-pulsar-%d", i);

        // Build a message object
        Message msg = MessageBuilder.create().setContent(content.getBytes()).build();

        // Send a message (waits until the message is persisted)
        MessageId msgId = producer.send(msg);

        log.info("Published msg='{}' with msg-id={}", content, msgId);
    }

    client.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchTime";
    final String subscriptionName = "time-sub-1" + compressionType.toString();

    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();

    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);

    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);

    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs,
            topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);

    producer.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime";
    final String subscriptionName = "time-size-sub-1" + compressionType.toString();

    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();

    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(5);
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);

    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);

    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs,
            topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);

    producer.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;

    // 1. producer connect
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);

    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        assertEquals(msg.getData(), ("my-message-" + i).getBytes());
    }

    // 3. producer disconnect
    producer.close();
    consumer.close();
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
@SuppressWarnings({ "unchecked", "deprecation" })
private PulsarKafkaProducer(Map<String, Object> conf, Properties properties, Serializer<K> keySerializer,
        Serializer<V> valueSerializer) {
    properties.forEach((k, v) -> conf.put((String) k, v));

    ProducerConfig producerConfig = new ProducerConfig(conf);

    if (keySerializer == null) {
        this.keySerializer = producerConfig.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
                Serializer.class);
        this.keySerializer.configure(producerConfig.originals(), true);
    } else {
        this.keySerializer = keySerializer;
        producerConfig.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
    }

    if (valueSerializer == null) {
        this.valueSerializer = producerConfig.getConfiguredInstance(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
                Serializer.class);
        this.valueSerializer.configure(producerConfig.originals(), true);
    } else {
        this.valueSerializer = valueSerializer;
        producerConfig.ignore(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
    }

    String serviceUrl = producerConfig.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG).get(0);
    ClientConfiguration clientConf = PulsarKafkaConfig.getClientConfiguration(properties);
    try {
        client = PulsarClient.create(serviceUrl, clientConf);
    } catch (PulsarClientException e) {
        throw new RuntimeException(e);
    }

    pulsarProducerConf = new ProducerConfiguration();
    pulsarProducerConf.setBatchingEnabled(true);

    // To mimic the same batching mode as Kafka, we need to wait a very little amount of
    // time to batch if the client is trying to send messages fast enough
    long lingerMs = Long.parseLong(properties.getProperty(ProducerConfig.LINGER_MS_CONFIG, "1"));
    pulsarProducerConf.setBatchingMaxPublishDelay(lingerMs, TimeUnit.MILLISECONDS);

    String compressionType = properties.getProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG);
    if ("gzip".equals(compressionType)) {
        pulsarProducerConf.setCompressionType(CompressionType.ZLIB);
    } else if ("lz4".equals(compressionType)) {
        pulsarProducerConf.setCompressionType(CompressionType.LZ4);
    }

    pulsarProducerConf.setSendTimeout(
            Integer.parseInt(properties.getProperty(ProducerConfig.MAX_BLOCK_MS_CONFIG, "60000")),
            TimeUnit.MILLISECONDS);

    boolean blockOnBufferFull = Boolean
            .parseBoolean(properties.getProperty(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG, "false"));

    // Kafka blocking semantic when blockOnBufferFull=false is different from Pulsar client
    // Pulsar throws error immediately when the queue is full and blockIfQueueFull=false
    // Kafka, on the other hand, still blocks for "max.block.ms" time and then gives error.
    boolean shouldBlockPulsarProducer = pulsarProducerConf.getSendTimeoutMs() > 0 || blockOnBufferFull;
    pulsarProducerConf.setBlockIfQueueFull(shouldBlockPulsarProducer);
}
 

import org.apache.pulsar.client.api.ProducerConfiguration; //导入方法依赖的package包/类
private ProducerConfiguration getProducerConfiguration() {
    ProducerConfiguration conf = new ProducerConfiguration();

    // Set to false to prevent the server thread from being blocked if a lot of messages are pending.
    conf.setBlockIfQueueFull(false);

    if (queryParams.containsKey("sendTimeoutMillis")) {
        conf.setSendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("batchingEnabled")) {
        conf.setBatchingEnabled(Boolean.parseBoolean(queryParams.get("batchingEnabled")));
    }

    if (queryParams.containsKey("batchingMaxMessages")) {
        conf.setBatchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages")));
    }

    if (queryParams.containsKey("maxPendingMessages")) {
        conf.setMaxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages")));
    }

    if (queryParams.containsKey("batchingMaxPublishDelay")) {
        conf.setBatchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")),
                TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("messageRoutingMode")) {
        checkArgument(
                Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(),
                "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode"));
        MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode"));
        if (!MessageRoutingMode.CustomPartition.equals(routingMode)) {
            conf.setMessageRoutingMode(routingMode);
        }
    }

    if (queryParams.containsKey("compressionType")) {
        checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(),
                "Invalid compressionType %s", queryParams.get("compressionType"));
        conf.setCompressionType(CompressionType.valueOf(queryParams.get("compressionType")));
    }

    return conf;
}