Java ProducerConfig類代碼示例

import kafka.producer.ProducerConfig; //導入依賴的package包/類
/**
 * 讀取配置文件，創建線程池，運行線程
 */
public void go() {
    Constant constant = new Constant();
    kafkaProperties kafkaProperties = new kafkaProperties();
    ProducerConfig config = new ProducerConfig(kafkaProperties.properties());

    ExecutorService executorService = Executors.newFixedThreadPool(Integer.parseInt(constant.THREAD_POOL_SIZE));

    String topic = constant.TOPIC_NAME;
    Task[] tasks = new Task[Integer.parseInt(constant.THREAD_NUM)];
    String[] folders = constant.FILE_FOLDERS.split(";");
    int batchSize = Integer.parseInt(constant.BATCH_SIZE);
    CopyOnWriteArrayList<String> fileList = addFiles(folders);

    for (int i = 0; i < tasks.length; ++i) {
        tasks[i] = new Task(i, topic, new Producer<String, String>(config), fileList, batchSize);
    }

    for (Task task : tasks) {
        executorService.execute(task);
    }
    executorService.shutdown();
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public static void produceMessages(String brokerList, String topic, int msgCount, String msgPayload) throws JSONException, IOException {
    
    // Add Producer properties and created the Producer
    ProducerConfig config = new ProducerConfig(setKafkaBrokerProps(brokerList));
    Producer<String, String> producer = new Producer<String, String>(config);

    LOG.info("KAFKA: Preparing To Send " + msgCount + " Events.");
    for (int i=0; i<msgCount; i++){

        // Create the JSON object
        JSONObject obj = new JSONObject();
        obj.put("id", String.valueOf(i));
        obj.put("msg", msgPayload);
        obj.put("dt", GenerateRandomDay.genRandomDay());
        String payload = obj.toString();

        KeyedMessage<String, String> data = new KeyedMessage<String, String>(topic, null, payload);
        producer.send(data);
        LOG.info("Sent message: " + data.toString());
    }
    LOG.info("KAFKA: Sent " + msgCount + " Events.");

    // Stop the producer
    producer.close();
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public KafkaProducer(){
    Properties props = new Properties();
    //此處配置的是kafka的端口
    props.put("metadata.broker.list", "192.168.1.116:9092");

    //配置value的序列化類
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    //配置key的序列化類
    props.put("key.serializer.class", "kafka.serializer.StringEncoder");

    //request.required.acks
    //0, which means that the producer never waits for an acknowledgement from the broker (the same behavior as 0.7). This option provides the lowest latency but the weakest durability guarantees (some data will be lost when a server fails).
    //1, which means that the producer gets an acknowledgement after the leader replica has received the data. This option provides better durability as the client waits until the server acknowledges the request as successful (only messages that were written to the now-dead leader but not yet replicated will be lost).
    //-1, which means that the producer gets an acknowledgement after all in-sync replicas have received the data. This option provides the best durability, we guarantee that no messages will be lost as long as at least one in sync replica remains.
    props.put("request.required.acks","-1");

    producer = new Producer<String, String>(new ProducerConfig(props));
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
private void connect(String serverURI, String clientId, String zkConnect) throws MqttException {
	
	mqtt = new MqttAsyncClient(serverURI, clientId);
	mqtt.setCallback(this);
	IMqttToken token = mqtt.connect();
	Properties props = new Properties();
	
	//Updated based on Kafka v0.8.1.1
	props.put("metadata.broker.list", "localhost:9092");
       props.put("serializer.class", "kafka.serializer.StringEncoder");
       props.put("partitioner.class", "example.producer.SimplePartitioner");
       props.put("request.required.acks", "1");
	
	ProducerConfig config = new ProducerConfig(props);
	kafkaProducer = new Producer<String, String>(config);
	token.waitForCompletion();
	logger.info("Connected to MQTT and Kafka");
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
private static Producer<String, String> buildAsyncProducer() {
	Properties props = new Properties();
	props.put("metadata.broker.list", Constants.BROKER_LIST);
	props.put("serializer.class", StringEncoder.class.getName());
	props.put("partitioner.class", HashPartitioner.class.getName());
	props.put("request.required.acks", "-1");
	props.put("producer.type", "async");  // 使用異步模式
	props.put("batch.num.messages", "3");  // 注意這裏會3個消息一起提交
	props.put("queue.buffer.max.ms", "10000000");
	props.put("queue.buffering.max.messages", "1000000");
	props.put("queue.enqueue.timeout.ms", "20000000");
	
	ProducerConfig config = new ProducerConfig(props);
	Producer<String, String> produce = new Producer<>(config);
	return produce;
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    //for backward compatibility.
    if(mapper == null) {
        this.mapper = new FieldNameBasedTupleToKafkaMapper<K,V>();
    }

    //for backward compatibility.
    if(topicSelector == null) {
        this.topicSelector = new DefaultTopicSelector((String) stormConf.get(TOPIC));
    }

    Map configMap = (Map) stormConf.get(KAFKA_BROKER_PROPERTIES);
    Properties properties = new Properties();
    properties.putAll(configMap);
    ProducerConfig config = new ProducerConfig(properties);
    producer = new Producer<K, V>(config);
    this.collector = collector;
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public static void main(String[] args) {
    Properties props = new Properties();
    props.put("metadata.broker.list", "127.0.0.1:9092");
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("key.serializer.class", "kafka.serializer.StringEncoder");
    props.put("request.required.acks","-1");

    Producer<String, String> producer = new Producer<String, String>(new ProducerConfig(props));

    int messageNo = 100;
    final int COUNT = 1000;
    while (messageNo < COUNT) {
        String key = String.valueOf(messageNo);
        String data = "hello kafka message " + key;
        producer.send(new KeyedMessage<String, String>("TestTopic", key ,data));
        System.out.println(data);
        messageNo ++;
    }
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
/**
 * constructor initializing
 * 
 * @param settings
 * @throws rrNvReadable.missingReqdSetting
 */
public KafkaPublisher(@Qualifier("propertyReader") rrNvReadable settings) throws rrNvReadable.missingReqdSetting {
	//fSettings = settings;

	final Properties props = new Properties();
	/*transferSetting(fSettings, props, "metadata.broker.list", "localhost:9092");
	transferSetting(fSettings, props, "request.required.acks", "1");
	transferSetting(fSettings, props, "message.send.max.retries", "5");
	transferSetting(fSettings, props, "retry.backoff.ms", "150"); */
	String kafkaConnUrl= com.att.ajsc.filemonitor.AJSCPropertiesMap.getProperty(CambriaConstants.msgRtr_prop,"kafka.metadata.broker.list"); 
	System.out.println("kafkaConnUrl:- "+kafkaConnUrl);
	if(null==kafkaConnUrl){ 

		kafkaConnUrl="localhost:9092"; 
	}		
	transferSetting( props, "metadata.broker.list", kafkaConnUrl);
	transferSetting( props, "request.required.acks", "1");
	transferSetting( props, "message.send.max.retries", "5");
	transferSetting(props, "retry.backoff.ms", "150"); 

	props.put("serializer.class", "kafka.serializer.StringEncoder");

	fConfig = new ProducerConfig(props);
	fProducer = new Producer<String, String>(fConfig);
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public static KafkaProducer getInstance(String brokerList) {
	long threadId = Thread.currentThread().getId();
	Producer<String, String> producer = _pool.get(threadId);
	System.out.println("producer:" + producer + ", thread:" + threadId);

	if (producer == null) {

		Preconditions.checkArgument(StringUtils.isNotBlank(brokerList), "kafka brokerList is blank...");

		// set properties
		Properties properties = new Properties();
		properties.put(METADATA_BROKER_LIST_KEY, brokerList);
		properties.put(SERIALIZER_CLASS_KEY, SERIALIZER_CLASS_VALUE);
		properties.put("kafka.message.CompressionCodec", "1");
		properties.put("client.id", "streaming-kafka-output");
		ProducerConfig producerConfig = new ProducerConfig(properties);

		producer = new Producer<String, String>(producerConfig);

		_pool.put(threadId, producer);
	}

	return instance;
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
/***
 * @param propManager     Used to get properties from json file
 * @param metaDataManager Used to get the broker list
 */
public Producer(IPropertiesManager<ProducerProperties> propManager, IMetaDataManager metaDataManager) throws MetaDataManagerException {
    m_metaDataManager = metaDataManager;
    m_propManager = propManager;
    producerProperties = m_propManager.getProperties();
    Properties props = new Properties();
    String brokerList = "";
    for (String broker : m_metaDataManager.getBrokerList(true)) {
        brokerList += broker + ", ";
    }
    props.put("metadata.broker.list", brokerList);
    props.put("serializer.class", producerProperties.serializer_class);
    props.put("partitioner.class", SimplePartitioner.class.getName());
    props.put("request.required.acks", producerProperties.request_required_acks.toString());

    ProducerConfig config = new ProducerConfig(props);
    m_producer = new kafka.javaapi.producer.Producer<String, String>(config);
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public static void main(String[] args) {
    Properties props = new Properties();
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("metadata.broker.list", "localhost:9092");

    Producer<String,String> producer = new Producer<String, String>(new ProducerConfig(props));

    int number = 1;
    for(; number < MESSAGES_NUMBER; number++)
    {
        String messageStr =
                String.format("{\"message\": %d, \"uid\":\"%s\"}",
                        number, uId.get(rand.nextInt(uNum)));

        producer.send(new KeyedMessage<String, String>(SparkStreamingConsumer.KAFKA_TOPIC,
                null, messageStr));
        if (number % 10000 == 0)
            System.out.println("Messages pushed: " + number);
    }
    System.out.println("Messages pushed: " + number);
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
private static Producer<String, String> initProducer() {
    Properties props = new Properties();
    props.put("metadata.broker.list", BROKER_LIST);
    // props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("serializer.class", StringEncoder.class.getName());
    props.put("partitioner.class", HashPartitioner.class.getName());
//    props.put("compression.codec", "0");
    props.put("producer.type", "sync");
    props.put("batch.num.messages", "1");
    props.put("queue.buffering.max.messages", "1000000");
    props.put("queue.enqueue.timeout.ms", "20000000");

    
    ProducerConfig config = new ProducerConfig(props);
    Producer<String, String> producer = new Producer<String, String>(config);
    return producer;
  }
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
protected void init() {
	if (properties == null) {
		properties = new Properties();
		try {
			properties.load(Thread.currentThread().getContextClassLoader()
					.getResourceAsStream(propertiesFile));
		} catch (IOException e) {
			log.error("The properties file is not loaded.", e);
			throw new IllegalArgumentException(
					"The properties file is not loaded.", e);
		}
	}
	log.info("Producer properties:" + properties);

	ProducerConfig config = new ProducerConfig(properties);
	producer = new Producer<String, String>(config);
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
public static void main(String[] args) throws Exception {
	//read config file
	Properties prop = PropertyFileReader.readPropertyFile();		
	String zookeeper = prop.getProperty("com.iot.app.kafka.zookeeper");
	String brokerList = prop.getProperty("com.iot.app.kafka.brokerlist");
	String topic = prop.getProperty("com.iot.app.kafka.topic");
	logger.info("Using Zookeeper=" + zookeeper + " ,Broker-list=" + brokerList + " and topic " + topic);

	// set producer properties
	Properties properties = new Properties();
	properties.put("zookeeper.connect", zookeeper);
	properties.put("metadata.broker.list", brokerList);
	properties.put("request.required.acks", "1");
	properties.put("serializer.class", "com.iot.app.kafka.util.IoTDataEncoder");
	//generate event
	Producer<String, IoTData> producer = new Producer<String, IoTData>(new ProducerConfig(properties));
	IoTDataProducer iotProducer = new IoTDataProducer();
	iotProducer.generateIoTEvent(producer,topic);		
}
 

import kafka.producer.ProducerConfig; //導入依賴的package包/類
private KafkaReporter(MetricRegistry registry, String name,
		TimeUnit rateUnit, TimeUnit durationUnit, boolean showSamples, MetricFilter filter,
		String topic, ProducerConfig config, String prefix,
		String hostName, String ip) {
	super(registry, name, filter, rateUnit, durationUnit);
	this.topic = topic;
	this.config = config;
	this.prefix = prefix;
	this.hostName = hostName;
	this.ip = ip;
	
	this.mapper = new ObjectMapper().registerModule(new MetricsModule(rateUnit,
               durationUnit,
               showSamples));

	producer = new Producer<String, String>(config);

	kafkaExecutor = Executors
			.newSingleThreadExecutor(new ThreadFactoryBuilder()
					.setNameFormat("kafka-producer-%d").build());
}