Java VerifiableProperties類代碼示例

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public void run() {
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(transducer_topic, new Integer(1));

    StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
    KafkaStream<String, String> stream = consumerMap.get(transducer_topic).get(0);
    ConsumerIterator<String, String> it = stream.iterator();
    while (it.hasNext() && bStartConsume){
        transducerDataProcessor.newData(it.next().message());

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public synchronized void init(VerifiableProperties props) {
    if (!initialized) {
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);
        influxDBAddress = props.getString("kafka.influxdb.metrics.address", INFLUXDB_DEFAULT_ADDRESS);
        influxDBUsername = props.getString("kafka.influxdb.metrics.username", INFLUXDB_DEFAULT_USERNAME);
        influxDBPassword = props.getString("kafka.influxdb.metrics.password", INFLUXDB_DEFAULT_PASSWORD);
        influxDBConsistency = props.getString("kafka.influxdb.metrics.consistency", INFLUXDB_DEFAULT_CONSISTENCY);
        influxDBDatabase = props.getString("kafka.influxdb.metrics.database", INFLUXDB_DEFAULT_DATABASE);
        influxDBRetentionPolicy = props.getString("kafka.influxdb.metrics.retentionPolicy", INFLUXDB_DEFAULT_RETENTIONPOLICY);
        influxDBTags = props.getString("kafka.influxdb.metrics.tags", INFLUXDB_DEFAULT_TAGS);
        metricDimensions = Dimension.fromProperties(props.props(), "kafka.influxdb.dimension.enabled.");

        LOG.debug("Initialize InfluxDBReporter [{},{},{}]", influxDBAddress, influxDBDatabase, influxDBRetentionPolicy);

        reporter = buildInfluxDBReporter();

        if (props.getBoolean("kafka.influxdb.metrics.reporter.enabled", false)) {
            initialized = true;
            startReporter(metricsConfig.pollingIntervalSecs());
            LOG.debug("InfluxDBReporter started.");
        }
    }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
void consume() throws Exception {
	// specify the number of consumer threads
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
	topicCountMap.put(KafkaProducer.TOPIC, new Integer(threadsNum));

	// specify data decoder
	StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
	StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

	Map<String, List<KafkaStream<String, String>>> consumerMap = consumer
			.createMessageStreams(topicCountMap, keyDecoder, valueDecoder); // 三個String分別為TOPIC、Key、Value

	// acquire data
	List<KafkaStream<String, String>> streams = consumerMap.get(KafkaProducer.TOPIC);

	// multi-threaded consume
	executor = Executors.newFixedThreadPool(threadsNum);    //create a thread pool
	for (final KafkaStream<String, String> stream : streams) {
		executor.submit(new ConsumerThread(stream));        // run thread
	}
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
public void collectMq(){
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
       topicCountMap.put(Constants.kfTopic, new Integer(1));

       StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
       StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

       Map<String, List<KafkaStream<String, String>>> consumerMap =
               consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
       
       KafkaStream<String, String> stream = consumerMap.get(Constants.kfTopic).get(0);
       ConsumerIterator<String, String> it = stream.iterator();
       MessageAndMetadata<String, String> msgMeta;
       while (it.hasNext()){
       	msgMeta = it.next();
       	super.mqTimer.parseMqText(msgMeta.key(), msgMeta.message());
       	//System.out.println(msgMeta.key()+"\t"+msgMeta.message());
       }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
private void consumeMessages() {
    final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(TOPIC, 1);
    final StringDecoder decoder =
            new StringDecoder(new VerifiableProperties());
    final Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap, decoder, decoder);
    final KafkaStream<String, String> stream =
            consumerMap.get(TOPIC).get(0);
    final ConsumerIterator<String, String> iterator = stream.iterator();

    Thread kafkaMessageReceiverThread = new Thread(
            () -> {
                while (iterator.hasNext()) {
                    String msg = iterator.next().message();
                    msg = msg == null ? "<null>" : msg;
                    System.out.println("got message: " + msg);
                    messagesReceived.add(msg);
                }
            },
            "kafkaMessageReceiverThread"
    );
    kafkaMessageReceiverThread.start();

}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public void init(VerifiableProperties props) {

    if(!initialized) {
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);

        InfluxDBMetricsConfig config = new InfluxDBMetricsConfig(props);
        config.addTag("brokerId", props.getString("broker.id"));

        this.reporter = new InfluxReporter(Metrics.defaultRegistry(), DEFAULT_NAME
                ,new InfluxDBClient(config), new MetricsPredicate(config.getPredicates()));

        if (props.getBoolean(InfluxDBMetricsConfig.KAFKA_INFLUX_METRICS_ENABLE, false)) {
            initialized = true;
            startReporter(metricsConfig.pollingIntervalSecs());
            LOG.info("KafkaInfluxMetricsReporter initialized.");
        }
    }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
    _collector = spoutOutputCollector;
    Properties props = new Properties();
    props.put("zookeeper.connect", conf.get(OSMIngest.ZOOKEEPERS));
    props.put("group.id", groupId);
    props.put("zookeeper.sync.time.ms", "200");
    props.put("auto.commit.interval.ms", "1000");
    ConsumerConfig consumerConfig = new ConsumerConfig(props);
    ConsumerConnector consumer = Consumer.createJavaConsumerConnector(consumerConfig);
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(topic, 1);
    Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, new StringDecoder(new VerifiableProperties()), new StringDecoder(new VerifiableProperties()));
    List<KafkaStream<String, String>> streams = consumerMap.get(topic);
    KafkaStream<String, String> stream = null;
    if (streams.size() == 1) {
        stream = streams.get(0);
    } else {
        log.error("Streams should be of size 1");
    }
    kafkaIterator = stream.iterator();
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Bean
protected KafkaStream<String, float[]> kafkaStream() {

    final String topicName = retrieveTopicNameFromGatewayAddress(gatewayUrl());

    ConsumerConnector consumerConnector =
            Consumer.createJavaConsumerConnector(consumerConfig());
    Map<String, Integer> topicCounts = new HashMap<>();
    topicCounts.put(topicName, 1);
    VerifiableProperties emptyProps = new VerifiableProperties();
    StringDecoder keyDecoder = new StringDecoder(emptyProps);
    FeatureVectorDecoder valueDecoder = new FeatureVectorDecoder();
    Map<String, List<KafkaStream<String, float[]>>> streams =
            consumerConnector.createMessageStreams(topicCounts, keyDecoder, valueDecoder);
    List<KafkaStream<String, float[]>> streamsByTopic = streams.get(topicName);
    Preconditions.checkNotNull(streamsByTopic, String.format("Topic %s not found in streams map.", topicName));
    Preconditions.checkElementIndex(0, streamsByTopic.size(),
            String.format("List of streams of topic %s is empty.", topicName));
    return streamsByTopic.get(0);
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Test
public void testKafkaLogAppender() {
    Properties consumerProps = new Properties();
    consumerProps.put("zookeeper.connect", zookeeper);
    consumerProps.put("group.id", "kafka-log-appender-test");
    consumerProps.put("auto.offset.reset", "smallest");
    consumerProps.put("schema.registry.url", schemaRegistry);

    Map<String, Integer> topicMap = new HashMap<String, Integer>();
    topicMap.put(topic, 1);

    ConsumerIterator<String, Object> iterator = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerProps))
            .createMessageStreams(topicMap, new StringDecoder(null), new KafkaAvroDecoder(new VerifiableProperties(consumerProps)))
            .get(topic).get(0).iterator();

    String testMessage = "I am a test message";
    logger.info(testMessage);

    MessageAndMetadata<String, Object> messageAndMetadata = iterator.next();
    GenericRecord logLine = (GenericRecord) messageAndMetadata.message();
    assertEquals(logLine.get("line").toString(), testMessage);
    assertEquals(logLine.get("logtypeid"), KafkaLogAppender.InfoLogTypeId);
    assertNotNull(logLine.get("source"));
    assertEquals(((Map<CharSequence, Object>) logLine.get("timings")).size(), 1);
    assertEquals(((Map<CharSequence, Object>) logLine.get("tag")).size(), 2);
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public AvroSerde getSerde(String s, Config config) {
    final String registryUrl = config.get(CFG_SCHEMA_REGISTRY_URL);
    if (registryUrl == null) {
        throw new ConfigException("Missing property: " + CFG_SCHEMA_REGISTRY_URL);
    }
    final String registryMasterUrl = config.get(CFG_SCHEMA_REGISTRY_MASTER_URL, registryUrl);
    final String specificReader = config.get(CFG_AVRO_SPECIFIC_DATA, "true");
    final Properties encoderProps = new Properties();
    encoderProps.setProperty("schema.registry.url", registryMasterUrl);
    logger.info("Avro encoder registry: " + registryMasterUrl);
    final Properties decoderProps = new Properties();
    decoderProps.setProperty("schema.registry.url", registryUrl);
    logger.info("Avro decoder registry: " + registryUrl);
    decoderProps.setProperty("specific.avro.reader", specificReader);
    return new AvroSerde(new VerifiableProperties(encoderProps), new VerifiableProperties(decoderProps));
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Test
public void testSerde() throws IOException {
  Properties props = new Properties();
  props.setProperty(KafkaAvroDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "bogus");
  props.setProperty(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, "true");
  AvroSerde avroSerde = new AvroSerde(new MockSchemaRegistryClient(), new VerifiableProperties(props));
  User user = User.newBuilder()
      .setName("Cornhoolio")
      .setAge(12)
      .build();

  byte[] bytes = avroSerde.toBytes(user);

  User userRead = (User) avroSerde.fromBytes(bytes);
  assertEquals(user, userRead);

  SimpleUser simpleUser = (SimpleUser) avroSerde.fromBytes(bytes, SimpleUser.getClassSchema());
  assertEquals("Cornhoolio", simpleUser.getName().toString());
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Test
public void testSerdeWithGenericData() throws IOException {
  Properties props = new Properties();
  props.setProperty(KafkaAvroDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "bogus");
  props.setProperty(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, "false");
  AvroSerde avroSerde = new AvroSerde(new MockSchemaRegistryClient(), new VerifiableProperties(props));
  User user = User.newBuilder()
      .setName("Cornhoolio")
      .setAge(12)
      .build();

  byte[] bytes = avroSerde.toBytes(user);

  GenericData.Record userRead = (GenericData.Record) avroSerde.fromBytes(bytes);
  assertNotNull(userRead.get("age"));

  GenericData.Record simpleUser = (GenericData.Record) avroSerde.fromBytes(bytes, SimpleUser.getClassSchema());
  //Projection masked the age field
  assertNull(simpleUser.get("age"));
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public void init(VerifiableProperties verifiableProperties) {

    if (!initialized) {
        // get configured metrics from kafka
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(verifiableProperties);

        // get the configured properties from kafka to set the bindAddress and port.
        bindAddress = verifiableProperties.getProperty("kafka.http.metrics.host");
        port = Integer.parseInt(verifiableProperties.getProperty("kafka.http.metrics.port"));
        enabled = Boolean.parseBoolean(verifiableProperties.getProperty("kafka.http.metrics.reporter.enabled"));

        // construct the Metrics Server
        metricsServer = new KafkaHttpMetricsServer(bindAddress, port);
        initialized = true;

        // call the method startReporter
        startReporter(metricsConfig.pollingIntervalSecs());
    } else {
        LOG.error("Kafka Http Metrics Reporter already initialized");
    }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
@Override
public synchronized void init(VerifiableProperties props) {
    if (!initialized) {
        KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);
        graphiteHost = props.getString("kafka.graphite.metrics.host", GRAPHITE_DEFAULT_HOST);
        graphitePort = props.getInt("kafka.graphite.metrics.port", GRAPHITE_DEFAULT_PORT);
        metricPrefix = props.getString("kafka.graphite.metrics.group", GRAPHITE_DEFAULT_PREFIX);
        String excludeRegex = props.getString("kafka.graphite.metrics.exclude.regex", null);
        metricDimensions = Dimension.fromProperties(props.props(), "kafka.graphite.dimension.enabled.");

        LOG.debug("Initialize GraphiteReporter [{},{},{}]", graphiteHost, graphitePort, metricPrefix);

        if (excludeRegex != null) {
            LOG.debug("Using regex [{}] for GraphiteReporter", excludeRegex);
            metricPredicate = new FilterMetricPredicate(excludeRegex);
        }
        reporter = buildGraphiteReporter();

        if (props.getBoolean("kafka.graphite.metrics.reporter.enabled", false)) {
            initialized = true;
            startReporter(metricsConfig.pollingIntervalSecs());
            LOG.debug("GraphiteReporter started.");
        }
    }
}
 

import kafka.utils.VerifiableProperties; //導入依賴的package包/類
private void setup(String groupId) {

	consumer = kafka.consumer.Consumer.createJavaConsumerConnector(
	            createConsumerConfig(groupId));
  
    // Request a single connection that gathers messages from all partitions 
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(topic, new Integer(1));
    
    // Construct the encoders/decoders for the message key and values
    Decoder<Long> keyDecoder = new LongSupport(new VerifiableProperties());
    Decoder<String> messageDecoder = new StringSupport(new VerifiableProperties());

	// Configure and request the desired streams using the topic map, encoder and decoder
    Map<String, List<KafkaStream<Long, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, keyDecoder, messageDecoder);

	// We only get back one stream
    KafkaStream<Long, String> stream =  consumerMap.get(topic).get(0);
    
    // Capture the stream's iterator
    it = stream.iterator();
  }