Java Decoder类代码示例

import kafka.serializer.Decoder; //导入依赖的package包/类
Unbound(Class<? extends Decoder<K>> keyDecoder,
        Class<? extends Decoder<V>> valueDecoder, Class<K> key,
        Class<V> value, Set<String> topics, Map<String, String> kafkaParams) {
  checkNotNull(keyDecoder, "need to set the key decoder class of a KafkaIO.Read transform");
  checkNotNull(
      valueDecoder, "need to set the value decoder class of a KafkaIO.Read transform");
  checkNotNull(key, "need to set the key class of a KafkaIO.Read transform");
  checkNotNull(value, "need to set the value class of a KafkaIO.Read transform");
  checkNotNull(topics, "need to set the topics of a KafkaIO.Read transform");
  checkNotNull(kafkaParams, "need to set the kafkaParams of a KafkaIO.Read transform");
  this.keyDecoderClass = keyDecoder;
  this.valueDecoderClass = valueDecoder;
  this.keyClass = key;
  this.valueClass = value;
  this.topics = topics;
  this.kafkaParams = kafkaParams;
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
KafkaConsumerPool(final int maxSize, final int expireAfterMins, final String brokers, final Decoder<K> keyMapper, final Decoder<V> valueMapper) {
    this.maxSize = maxSize;
    this.expireAfterMins = expireAfterMins;
    this.brokerList = KafkaMetaData.createBrokerList(brokers);
    this.keyMapper = keyMapper;
    this.valueMapper = valueMapper;
    this.pool = createPool();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                LOG.info("Shutdown Hook: Shutting down KafkaConsumerPool");
                shutdown();
            } catch (Exception e) {
                LOG.error(e.getMessage(), e);
            }
        }
    });
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
public void run(Decoder<K> keyDecoder, Decoder<V> valueDecoder){
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
       topicCountMap.put(topic, threadNum);
       Map<String, List<KafkaStream<K, V>>> consumerMap = consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
       
       List<KafkaStream<K, V>> streams = consumerMap.get(topic);
       
       executor = Executors.newFixedThreadPool(threadNum);
    
       int threadNo = 0;
	for (final KafkaStream<K, V> stream : streams) {
       	ConsumerWorker<K, V> worker = new ConsumerWorker<K, V>(stream, threadNo);
       	executor.submit(worker);
       	threadNo++;
       }
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
private static <K, V> TransformEvaluator<KafkaIO.Read.Unbound<K, V>> kafka() {
  return new TransformEvaluator<KafkaIO.Read.Unbound<K, V>>() {
    @Override
    public void evaluate(KafkaIO.Read.Unbound<K, V> transform, EvaluationContext context) {
      StreamingEvaluationContext sec = (StreamingEvaluationContext) context;
      JavaStreamingContext jssc = sec.getStreamingContext();
      Class<K> keyClazz = transform.getKeyClass();
      Class<V> valueClazz = transform.getValueClass();
      Class<? extends Decoder<K>> keyDecoderClazz = transform.getKeyDecoderClass();
      Class<? extends Decoder<V>> valueDecoderClazz = transform.getValueDecoderClass();
      Map<String, String> kafkaParams = transform.getKafkaParams();
      Set<String> topics = transform.getTopics();
      JavaPairInputDStream<K, V> inputPairStream = KafkaUtils.createDirectStream(jssc, keyClazz,
              valueClazz, keyDecoderClazz, valueDecoderClazz, kafkaParams, topics);
      JavaDStream<WindowedValue<KV<K, V>>> inputStream =
          inputPairStream.map(new Function<Tuple2<K, V>, KV<K, V>>() {
        @Override
        public KV<K, V> call(Tuple2<K, V> t2) throws Exception {
          return KV.of(t2._1(), t2._2());
        }
      }).map(WindowingHelpers.<KV<K, V>>windowFunction());
      sec.setStream(transform, inputStream);
    }
  };
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
Unbound(Class<? extends Decoder<K>> keyDecoder,
        Class<? extends Decoder<V>> valueDecoder, Class<K> key,
        Class<V> value, Set<String> topics, Map<String, String> kafkaParams) {
  Preconditions.checkNotNull(keyDecoder,
      "need to set the key decoder class of a KafkaIO.Read transform");
  Preconditions.checkNotNull(valueDecoder,
      "need to set the value decoder class of a KafkaIO.Read transform");
  Preconditions.checkNotNull(key,
      "need to set the key class of aKafkaIO.Read transform");
  Preconditions.checkNotNull(value,
      "need to set the value class of a KafkaIO.Read transform");
  Preconditions.checkNotNull(topics,
      "need to set the topics of a KafkaIO.Read transform");
  Preconditions.checkNotNull(kafkaParams,
      "need to set the kafkaParams of a KafkaIO.Read transform");
  this.keyDecoderClass = keyDecoder;
  this.valueDecoderClass = valueDecoder;
  this.keyClass = key;
  this.valueClass = value;
  this.topics = topics;
  this.kafkaParams = kafkaParams;
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
private void setup(String groupId) {

	consumer = kafka.consumer.Consumer.createJavaConsumerConnector(
	            createConsumerConfig(groupId));
  
    // Request a single connection that gathers messages from all partitions 
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(topic, new Integer(1));
    
    // Construct the encoders/decoders for the message key and values
    Decoder<Long> keyDecoder = new LongSupport(new VerifiableProperties());
    Decoder<String> messageDecoder = new StringSupport(new VerifiableProperties());

	// Configure and request the desired streams using the topic map, encoder and decoder
    Map<String, List<KafkaStream<Long, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, keyDecoder, messageDecoder);

	// We only get back one stream
    KafkaStream<Long, String> stream =  consumerMap.get(topic).get(0);
    
    // Capture the stream's iterator
    it = stream.iterator();
  }
 

import kafka.serializer.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Test
public void createStreams_withoutTopicFilterRegex_isOk() {
    // Given
    final ConsumerFactory consumerFactory = new ConsumerFactory(createDefaultConsumerConfig());
    final ConsumerConnector consumer = mock(ConsumerConnector.class);
    when(consumer.createMessageStreamsByFilter(any(TopicFilter.class), anyInt(), any(Decoder.class), any(Decoder.class))).thenReturn(new ArraySeq(0));

    // When
    consumerFactory.createStreams(1, consumer);

    // Then
    verify(consumer).createMessageStreamsByFilter(
            refEq(new Whitelist(".*")), anyInt(), any(Decoder.class), any(Decoder.class)
    );
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Test
public void createStreams_withTopicFilterRegex_isOk() {
    // Given
    final String rawRegex = "foo.*";
    final ConsumerConfig consumerConfig = createDefaultConsumerConfig();
    consumerConfig.props().props().setProperty(ConsumerFactory.CONSUMER_TOPIC_FILTER_REGEX, rawRegex);

    final ConsumerFactory consumerFactory = new ConsumerFactory(consumerConfig);
    final ConsumerConnector consumer = mock(ConsumerConnector.class);
    when(consumer.createMessageStreamsByFilter(any(TopicFilter.class), anyInt(), any(Decoder.class), any(Decoder.class))).thenReturn(new ArraySeq(0));

    // When
    consumerFactory.createStreams(1, consumer);

    // Then
    verify(consumer).createMessageStreamsByFilter(
            refEq(new Whitelist(rawRegex)), anyInt(), any(Decoder.class), any(Decoder.class)
    );
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
KafkaConsumerService(MetricRegistry metrics,
                     ConsumerConfig consumerConfig,
                     Map<String, Integer> topics,
                     Decoder<K> keyDecoder,
                     Decoder<V> valueDecoder,
                     MessageHandler<K, V> messageHandler) {

    MDC.put("group_id", consumerConfig.groupId());

    this.metrics = metrics;
    this.consumerConfig = consumerConfig;
    this.topics = topics;
    this.keyDecoder = keyDecoder;
    this.valueDecoder = valueDecoder;
    this.messageHandler = messageHandler;
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public final Decoder<K> createKeyDecoder() {
    try {
        return (Decoder<K>)keyMapper.getDeclaredConstructor(VerifiableProperties.class).newInstance(verifiableProperties != null ? new VerifiableProperties(verifiableProperties) : null);
    } catch (Exception e) {
        throw new IllegalArgumentException("Unable to create key decoder: " + e.getMessage(), e);
    }
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public final Decoder<V> createValueDecoder() {
    try {
        return (Decoder<V>)valueMapper.getDeclaredConstructor(VerifiableProperties.class).newInstance(verifiableProperties != null ? new VerifiableProperties(verifiableProperties) : null);
    } catch (Exception e) {
        throw new IllegalArgumentException("Unable to create value decoder: " + e.getMessage(), e);
    }
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
KafkaConsumerFactory(final List<Broker> brokerList,
                     final Decoder<K> keyMapper,
                     final Decoder<V> valueMapper,
                     final String topic,
                     final Integer partition) {
    this.brokerList = brokerList;
    this.keyMapper = keyMapper;
    this.valueMapper = valueMapper;
    this.topic = topic;
    this.partition = partition;
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
public KafkaConsumer(final List<Broker> brokerList, final String consumerGroup,
                     final Decoder<K> keyDecoder, final Decoder<V> valueDecoder,
                     final String topic, final int partitionId) {
    this.brokersList = brokerList;
    this.consumerGroup = consumerGroup;
    this.keyDecoder = keyDecoder;
    this.valueDecoder = valueDecoder;
    this.topic = topic;
    this.partitionId = partitionId;

    this.clientId = KafkaMetaData.createClientId(consumerGroup, topic, partitionId);
    this.leaderBrokerChecker = new LeaderBrokerChecker(brokersList, topic, partitionId);

    connect();
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
public KafkaConsumer(final String topic, final String zkConnect, final String groupId, Decoder<Val> decoder){
	
	 
	 consumer = kafka.consumer.Consumer.createJavaConsumerConnector(
				new ConsumerConfig(getConsumerConfig(zkConnect, groupId)));
	 
	 Map<String, Integer> topicCountMap = new HashMap(){{
            put(topic, new Integer(1));
	 }};
	 Map<String, List<KafkaStream<byte[], Val>>> consumerMap = consumer.createMessageStreams(topicCountMap, new DefaultDecoder(null), decoder);
	 stream = consumerMap.get(topic).get(0);
}
 

import kafka.serializer.Decoder; //导入依赖的package包/类
/**
 * Constructor used for testing.
 */
public KafkaGenericAvroMutationDecoder(Decoder<byte[]> _firstDeserializer) {
    //TODO: Move to AbstractAvroDeserilaizer
    if (_firstDeserializer == null) {
        firstDeserializer = new DefaultDecoder(null);
        //TODO: Should get this from config - look at KafkaProducer code...(in Kafka src)
    } else {
        firstDeserializer = _firstDeserializer;
    }

}