Java ExtendedProducerProperties.setPartitionCount方法代码示例

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(10);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(10);
	producerProperties.setPartitionKeyExpression(new LiteralExpression("foo"));

	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	Binding<MessageChannel> producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0",
			moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null,
			moduleInputChannel, consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload)
			.build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(10);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(6);
	Binder binder = getBinder(binderConfiguration);
	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(5);
	producerProperties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload"));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	Binding<MessageChannel> producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0",
			moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null,
			moduleInputChannel, consumerProperties);
	Thread.sleep(1000);
	Message<?> message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload)
			.build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);

	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(6);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testCustomPartitionCountOverridesPartitioningIfLarger() throws Exception {
	byte[] testPayload = new byte[2048];
	Arrays.fill(testPayload, (byte) 65);
	KafkaBinderConfigurationProperties binderConfiguration = createConfigurationProperties();
	binderConfiguration.setMinPartitionCount(4);
	Binder binder = getBinder(binderConfiguration);

	QueueChannel moduleInputChannel = new QueueChannel();
	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionCount(5);
	producerProperties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload"));
	DirectChannel moduleOutputChannel = createBindableChannel("output",
			createProducerBindingProperties(producerProperties));
	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	long uniqueBindingId = System.currentTimeMillis();
	Binding<MessageChannel> producerBinding = binder.bindProducer("foo" + uniqueBindingId + ".0",
			moduleOutputChannel, producerProperties);
	Binding<MessageChannel> consumerBinding = binder.bindConsumer("foo" + uniqueBindingId + ".0", null,
			moduleInputChannel, consumerProperties);
	Message<?> message = org.springframework.integration.support.MessageBuilder.withPayload(testPayload)
			.build();
	// Let the consumer actually bind to the producer before sending a msg
	binderBindUnbindLatency();
	moduleOutputChannel.send(message);
	Message<?> inbound = receive(moduleInputChannel);
	assertThat(inbound).isNotNull();
	assertThat((byte[]) inbound.getPayload()).containsExactly(testPayload);
	assertThat(partitionSize("foo" + uniqueBindingId + ".0")).isEqualTo(5);
	producerBinding.unbind();
	consumerBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@Override
public void testPartitionedModuleJava() throws Exception {
	KinesisTestBinder binder = getBinder();

	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);

	final List<Message<?>> results = new ArrayList<>();
	final CountDownLatch receiveLatch = new CountDownLatch(3);

	MessageHandler receivingHandler = message -> {
		results.add(message);
		receiveLatch.countDown();
	};

	DirectChannel input0 = createBindableChannel("test.input0J", new BindingProperties());
	input0.subscribe(receivingHandler);

	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.0", "testPartitionedModuleJava", input0,
			consumerProperties);

	consumerProperties.setInstanceIndex(1);

	DirectChannel input1 = createBindableChannel("test.input1J", new BindingProperties());
	input1.subscribe(receivingHandler);

	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.0", "testPartitionedModuleJava", input1,
			consumerProperties);

	consumerProperties.setInstanceIndex(2);

	DirectChannel input2 = createBindableChannel("test.input2J", new BindingProperties());
	input2.subscribe(receivingHandler);

	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.0", "testPartitionedModuleJava", input2,
			consumerProperties);

	ExtendedProducerProperties<KinesisProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionKeyExtractorClass(PartitionTestSupport.class);
	producerProperties.setPartitionSelectorClass(PartitionTestSupport.class);
	producerProperties.setPartitionCount(3);

	DirectChannel output = createBindableChannel("test.output",
			createProducerBindingProperties(producerProperties));

	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.0", output, producerProperties);
	if (usesExplicitRouting()) {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("partJ.0", "testPartitionedModuleJava")
						+ "-' + headers['" + BinderHeaders.PARTITION_HEADER + "']");
	}

	output.send(new GenericMessage<>(2));
	output.send(new GenericMessage<>(1));
	output.send(new GenericMessage<>(0));

	assertThat(receiveLatch.await(20, TimeUnit.SECONDS)).isTrue();

	assertThat(results).extracting("payload").containsExactlyInAnyOrder("0", "1", "2");

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Override
protected MessageHandler createProducerMessageHandler(final ProducerDestination destination,
		ExtendedProducerProperties<KafkaProducerProperties> producerProperties, MessageChannel errorChannel)
		throws Exception {
	/*
	 * IMPORTANT: With a transactional binder, individual producer properties for Kafka are
	 * ignored; the global binder (spring.cloud.stream.kafka.binder.transaction.producer.*)
	 * properties are used instead, for all producers. A binder is transactional when
	 * 'spring.cloud.stream.kafka.binder.transaction.transaction-id-prefix' has text.
	 */
	final ProducerFactory<byte[], byte[]> producerFB = this.transactionManager != null
			? this.transactionManager.getProducerFactory()
			: getProducerFactory(null, producerProperties);
	Collection<PartitionInfo> partitions = provisioningProvider.getPartitionsForTopic(
			producerProperties.getPartitionCount(), false,
			() -> {
				Producer<byte[], byte[]> producer = producerFB.createProducer();
				List<PartitionInfo> partitionsFor = producer.partitionsFor(destination.getName());
				producer.close();
				((DisposableBean) producerFB).destroy();
				return partitionsFor;
			});
	this.topicsInUse.put(destination.getName(), new TopicInformation(null, partitions));
	if (producerProperties.getPartitionCount() < partitions.size()) {
		if (this.logger.isInfoEnabled()) {
			this.logger.info("The `partitionCount` of the producer for topic " + destination.getName() + " is "
					+ producerProperties.getPartitionCount() + ", smaller than the actual partition count of "
					+ partitions.size() + " of the topic. The larger number will be used instead.");
		}
		/*
		 * This is dirty; it relies on the fact that we, and the partition interceptor, share a
		 * hard reference to the producer properties instance. But I don't see another way to fix
		 * it since the interceptor has already been added to the channel, and we don't have
		 * access to the channel here; if we did, we could inject the proper partition count
		 * there. TODO: Consider this when doing the 2.0 binder restructuring.
		 */
		producerProperties.setPartitionCount(partitions.size());
	}

	KafkaTemplate<byte[], byte[]> kafkaTemplate = new KafkaTemplate<>(producerFB);
	if (this.producerListener != null) {
		kafkaTemplate.setProducerListener(this.producerListener);
	}
	ProducerConfigurationMessageHandler handler = new ProducerConfigurationMessageHandler(kafkaTemplate,
			destination.getName(), producerProperties, producerFB);
	if (errorChannel != null) {
		handler.setSendFailureChannel(errorChannel);
	}
	KafkaHeaderMapper mapper = null;
	if (this.configurationProperties.getHeaderMapperBeanName() != null) {
		mapper = getApplicationContext().getBean(this.configurationProperties.getHeaderMapperBeanName(),
				KafkaHeaderMapper.class);
	}
	/*
	 *  Even if the user configures a bean, we must not use it if the header
	 *  mode is not the default (headers); setting the mapper to null
	 *  disables populating headers in the message handler.
	 */
	if (producerProperties.getHeaderMode() != null
			&& !HeaderMode.headers.equals(producerProperties.getHeaderMode())) {
		mapper = null;
	}
	else if (mapper == null) {
		String[] headerPatterns = producerProperties.getExtension().getHeaderPatterns();
		if (headerPatterns != null && headerPatterns.length > 0) {
			List<String> patterns = new LinkedList<>(Arrays.asList(headerPatterns));
			if (!patterns.contains("!" + MessageHeaders.TIMESTAMP)) {
				patterns.add(0, "!" + MessageHeaders.TIMESTAMP);
			}
			if (!patterns.contains("!" + MessageHeaders.ID)) {
				patterns.add(0, "!" + MessageHeaders.ID);
			}
			mapper = new DefaultKafkaHeaderMapper(patterns.toArray(new String[patterns.size()]));
		}
		else {
			mapper = new DefaultKafkaHeaderMapper();
		}
	}
	handler.setHeaderMapper(mapper);
	return handler;
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void testPartitionedModuleJava() throws Exception {
	Binder binder = getBinder();

	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(4);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0J");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.0", "test", input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1J");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.0", "test", input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2J");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.0", "test", input2, consumerProperties);
	consumerProperties.setInstanceIndex(3);
	QueueChannel input3 = new QueueChannel();
	input3.setBeanName("test.input3J");
	Binding<MessageChannel> input3Binding = binder.bindConsumer("partJ.0", "test", input3, consumerProperties);

	ExtendedProducerProperties<KafkaProducerProperties> producerProperties = createProducerProperties();
	producerProperties.setPartitionKeyExtractorClass(PartitionTestSupport.class);
	producerProperties.setPartitionSelectorClass(PartitionTestSupport.class);
	producerProperties.setPartitionCount(3); // overridden to 8 on the actual topic
	DirectChannel output = createBindableChannel("output", createProducerBindingProperties(producerProperties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.0", output, producerProperties);
	if (usesExplicitRouting()) {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("partJ.0", "test") + "-' + headers['partition']");
	}

	output.send(new GenericMessage<>(2));
	output.send(new GenericMessage<>(1));
	output.send(new GenericMessage<>(0));
	output.send(new GenericMessage<>(3));

	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();
	Message<?> receive3 = receive(input3);
	assertThat(receive3).isNotNull();
	ObjectMapper om = new ObjectMapper();

	assertThat(om.readValue((byte[]) receive0.getPayload(), Integer.class)).isEqualTo(0);
	assertThat(om.readValue((byte[]) receive1.getPayload(), Integer.class)).isEqualTo(1);
	assertThat(om.readValue((byte[]) receive2.getPayload(), Integer.class)).isEqualTo(2);
	assertThat(om.readValue((byte[]) receive3.getPayload(), Integer.class)).isEqualTo(3);

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	input3Binding.unbind();
	outputBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testPartitionedModuleJavaWithRawMode() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> properties = createProducerProperties();
	properties.setHeaderMode(HeaderMode.none);
	properties.setPartitionKeyExtractorClass(RawKafkaPartitionTestSupport.class);
	properties.setPartitionSelectorClass(RawKafkaPartitionTestSupport.class);
	properties.setPartitionCount(6);

	DirectChannel output = createBindableChannel("output", createProducerBindingProperties(properties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("partJ.raw.0", output, properties);

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setPartitioned(true);
	consumerProperties.setHeaderMode(HeaderMode.none);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0J");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("partJ.raw.0", "test", input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1J");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("partJ.raw.0", "test", input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2J");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("partJ.raw.0", "test", input2, consumerProperties);

	output.send(new GenericMessage<>(new byte[]{(byte) 0}));
	output.send(new GenericMessage<>(new byte[]{(byte) 1}));
	output.send(new GenericMessage<>(new byte[]{(byte) 2}));

	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();

	assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], ((byte[]) receive1.getPayload())[0],
			((byte[]) receive2.getPayload())[0])).containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2);

	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}
 

import org.springframework.cloud.stream.binder.ExtendedProducerProperties; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testPartitionedModuleSpELWithRawMode() throws Exception {
	Binder binder = getBinder();
	ExtendedProducerProperties<KafkaProducerProperties> properties = createProducerProperties();
	properties.setPartitionKeyExpression(spelExpressionParser.parseExpression("payload[0]"));
	properties.setPartitionSelectorExpression(spelExpressionParser.parseExpression("hashCode()"));
	properties.setPartitionCount(6);
	properties.setHeaderMode(HeaderMode.none);

	DirectChannel output = createBindableChannel("output", createProducerBindingProperties(properties));
	output.setBeanName("test.output");
	Binding<MessageChannel> outputBinding = binder.bindProducer("part.raw.0", output, properties);
	try {
		Object endpoint = extractEndpoint(outputBinding);
		assertThat(getEndpointRouting(endpoint))
				.contains(getExpectedRoutingBaseDestination("part.raw.0", "test") + "-' + headers['partition']");
	}
	catch (UnsupportedOperationException ignored) {
	}

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	consumerProperties.setConcurrency(2);
	consumerProperties.setInstanceIndex(0);
	consumerProperties.setInstanceCount(3);
	consumerProperties.setPartitioned(true);
	consumerProperties.setHeaderMode(HeaderMode.none);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	QueueChannel input0 = new QueueChannel();
	input0.setBeanName("test.input0S");
	Binding<MessageChannel> input0Binding = binder.bindConsumer("part.raw.0", "test", input0, consumerProperties);
	consumerProperties.setInstanceIndex(1);
	QueueChannel input1 = new QueueChannel();
	input1.setBeanName("test.input1S");
	Binding<MessageChannel> input1Binding = binder.bindConsumer("part.raw.0", "test", input1, consumerProperties);
	consumerProperties.setInstanceIndex(2);
	QueueChannel input2 = new QueueChannel();
	input2.setBeanName("test.input2S");
	Binding<MessageChannel> input2Binding = binder.bindConsumer("part.raw.0", "test", input2, consumerProperties);

	Message<byte[]> message2 = org.springframework.integration.support.MessageBuilder.withPayload(new byte[]{2})
			.setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "kafkaBinderTestCommonsDelegate")
			.setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42)
			.setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43).build();
	output.send(message2);
	output.send(new GenericMessage<>(new byte[]{1}));
	output.send(new GenericMessage<>(new byte[]{0}));
	Message<?> receive0 = receive(input0);
	assertThat(receive0).isNotNull();
	Message<?> receive1 = receive(input1);
	assertThat(receive1).isNotNull();
	Message<?> receive2 = receive(input2);
	assertThat(receive2).isNotNull();
	assertThat(Arrays.asList(((byte[]) receive0.getPayload())[0], ((byte[]) receive1.getPayload())[0],
			((byte[]) receive2.getPayload())[0])).containsExactlyInAnyOrder((byte) 0, (byte) 1, (byte) 2);
	input0Binding.unbind();
	input1Binding.unbind();
	input2Binding.unbind();
	outputBinding.unbind();
}