當前位置: 首頁>>代碼示例>>Java>>正文

Java ConsumerIterator類代碼示例

本文整理匯總了Java中kafka.consumer.ConsumerIterator類的典型用法代碼示例。如果您正苦於以下問題：Java ConsumerIterator類的具體用法？Java ConsumerIterator怎麽用？Java ConsumerIterator使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


ConsumerIterator類屬於kafka.consumer包，在下文中一共展示了ConsumerIterator類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
/**
 * When an object implementing interface <code>Runnable</code> is used
 * to create a thread, starting the thread causes the object's
 * <code>run</code> method to be called in that separately executing
 * thread.
 * <p>
 * The general contract of the method <code>run</code> is that it may
 * take any action whatsoever.
 *
 * @see Thread#run()
 */
@Override
public void run() {
    ConsumerConnector consumerConnector = KafkaUtils.createConsumerConnector(zkAddr, group);
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(CONSUMER_OFFSET_TOPIC, new Integer(1));
    KafkaStream<byte[], byte[]> offsetMsgStream = consumerConnector.createMessageStreams(topicCountMap).get(CONSUMER_OFFSET_TOPIC).get(0);

    ConsumerIterator<byte[], byte[]> it = offsetMsgStream.iterator();
    while (true) {

        MessageAndMetadata<byte[], byte[]> offsetMsg = it.next();
        if (ByteBuffer.wrap(offsetMsg.key()).getShort() < 2) {
            try {
                GroupTopicPartition commitKey = readMessageKey(ByteBuffer.wrap(offsetMsg.key()));
                if (offsetMsg.message() == null) {
                    continue;
                }
                kafka.common.OffsetAndMetadata commitValue = readMessageValue(ByteBuffer.wrap(offsetMsg.message()));
                kafkaConsumerOffsets.put(commitKey, commitValue);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}
開發者ID:dubin555，項目名稱:Kafka-Insight，代碼行數:37，代碼來源:KafkaOffsetGetter.java

示例2: getNextMessage

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public MessageAndMetadata getNextMessage(String topic) {
  List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);
  // it has only a single stream, because there is only one consumer
  KafkaStream stream = streams.get(0);
  final ConsumerIterator<byte[], byte[]> it = stream.iterator();
  int counter = 0;
  try {
    if (it.hasNext()) {
      return it.next();
    } else {
      return null;
    }
  } catch (ConsumerTimeoutException e) {
    logger.error("0 messages available to fetch for the topic " + topic);
    return null;
  }
}
開發者ID:moueimei，項目名稱:flume-release-1.7.0，代碼行數:18，代碼來源:KafkaConsumer.java

示例3: nextTuple

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public void nextTuple() {
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
	topicCountMap.put(TopologyConfig.kafkaTopic, 1);//one excutor - one thread
	Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = conn.createMessageStreams(topicCountMap);
	List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(kafkaTopic);
	ConsumerIterator<byte[], byte[]> iter = streams.get(0).iterator();
	while(true){
		while(iter.hasNext()){
			
			String s = new String(iter.next().message());
			collector.emit(new Values(s));
			 
			UUID msgId = UUID.randomUUID();
			this.pending.put(msgId, new Values(s));
		}
		try {
			Thread.sleep(1000L);
		} catch (InterruptedException e) {
			logger.error("Spout : sleep wrong \n", e);
		}
	}
}
開發者ID:zhai3516，項目名稱:storm-demos，代碼行數:23，代碼來源:KafkaDataSpout.java

示例4: processStreamsByTopic

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
private void processStreamsByTopic(String topicKeys, List<KafkaStream<byte[], byte[]>> streamList) {
    // init stream thread pool
    ExecutorService streamPool = Executors.newFixedThreadPool(partitions);
    String[] topics = StringUtils.split(topicKeys, ",");
    if (log.isDebugEnabled())
        log.debug("準備處理消息流集合 KafkaStreamList,topic count={},topics={}, partitions/topic={}", topics.length, topicKeys, partitions);

    //遍曆stream
    AtomicInteger index = new AtomicInteger(0);
    for (KafkaStream<byte[], byte[]> stream : streamList) {
        Thread streamThread = new Thread() {

            @Override
            public void run() {
                int i = index.getAndAdd(1);
                if (log.isDebugEnabled())
                    log.debug("處理消息流KafkaStream -- No.={}, partitions={}", i, partitions + ":" + i);

                ConsumerIterator<byte[], byte[]> consumerIterator = stream.iterator();

                processStreamByConsumer(topicKeys, consumerIterator);
            }
        };
        streamPool.execute(streamThread);
    }
}
開發者ID:KoperGroup，項目名稱:koper，代碼行數:27，代碼來源:KafkaReceiver.java

示例5: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public void run() {
	try {
		while(true){
			ConsumerIterator<byte[], byte[]> it = m_stream.iterator();
			while (it.hasNext()) {
				String m = null;
				try {
					m = new String(it.next().message(),
							this.kafkaInput.encoding);
					Map<String, Object> event = this.decoder
							.decode(m);
					if(zkDistributed==null){
						this.kafkaInput.process(event);
					}else{
						zkDistributed.route(event);
					}
				} catch (Exception e) {
					logger.error("process event:{} failed:{}",m,ExceptionUtil.getErrorMessage(e));
				}
			}
		}
	} catch (Exception t) {
		logger.error("kakfa Consumer fetch is error:{}",ExceptionUtil.getErrorMessage(t));
	}
}
開發者ID:DTStack，項目名稱:jlogstash-input-plugin，代碼行數:26，代碼來源:KafkaDistributed.java

示例6: shouldWriteThenRead

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
@Test
public void shouldWriteThenRead() throws Exception {

    //Create a consumer
    ConsumerIterator<String, String> it = buildConsumer(Original.topic);

    //Create a producer
    producer = new KafkaProducer<>(producerProps());

    //send a message
    producer.send(new ProducerRecord<>(Original.topic, "message")).get();

    //read it back
    MessageAndMetadata<String, String> messageAndMetadata = it.next();
    String value = messageAndMetadata.message();
    assertThat(value, is("message"));
}
開發者ID:telstra，項目名稱:open-kilda，代碼行數:18，代碼來源:Original.java

示例7: shouldWriteThenRead

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
@Test
public void shouldWriteThenRead() throws Exception {

    //Create a consumer
    ConsumerIterator<String, String> it = buildConsumer(SimpleKafkaTest.topic);

    //Create a producer
    producer = new KafkaProducer<>(producerProps());

    //send a message
    producer.send(new ProducerRecord<>(SimpleKafkaTest.topic, "message")).get();

    //read it back
    MessageAndMetadata<String, String> messageAndMetadata = it.next();
    String value = messageAndMetadata.message();
    assertThat(value, is("message"));
}
開發者ID:telstra，項目名稱:open-kilda，代碼行數:18，代碼來源:SimpleKafkaTest.java

示例8: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
@Override
public void run() {
    ConsumerIterator<byte[], byte[]> it = stream.iterator();
    while (it.hasNext()) {
        MessageAndMetadata<byte[], byte[]> mam = it.next();
        String jsonStr = "";
        try {
            jsonStr = new String(mam.message());
            JSONObject jsonObject = JSONObject.parseObject(jsonStr);
            LogcenterConfig config = LogConfigCache.getLogConfigCache(jsonObject);
            IStorageApi iStorageApi = ServiceRegister.getInstance().getProvider(config.getStorageType());
            iStorageApi.save(jsonObject);
        } catch (Exception e) {
            e.printStackTrace();
            logger.error("partition[" + mam.partition() + "]," + "offset[" + mam.offset() + "], " + jsonStr, e);
            continue;
        }
    }
}
開發者ID:geeker-lait，項目名稱:tasfe-framework，代碼行數:20，代碼來源:KafkaConsumerThread.java

示例9: testLogging

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
@Test
public void testLogging() throws InterruptedException {

    for (int i = 0; i<1000; ++i) {
        logger.info("message"+i);
    }

    final KafkaStream<byte[], byte[]> log = kafka.createClient().createMessageStreamsByFilter(new Whitelist("logs"),1).get(0);
    final ConsumerIterator<byte[], byte[]> iterator = log.iterator();

    for (int i=0; i<1000; ++i) {
        final String messageFromKafka = new String(iterator.next().message(), UTF8);
        assertThat(messageFromKafka, Matchers.equalTo("message"+i));
    }

}
開發者ID:wngn123，項目名稱:wngn-jms-kafka，代碼行數:17，代碼來源:LogbackIntegrationIT.java

示例10: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
@Override
public void run() {
    Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(transducer_topic, new Integer(1));

    StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
    KafkaStream<String, String> stream = consumerMap.get(transducer_topic).get(0);
    ConsumerIterator<String, String> it = stream.iterator();
    while (it.hasNext() && bStartConsume){
        transducerDataProcessor.newData(it.next().message());

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
開發者ID:unrealinux，項目名稱:DataProcessPlatformKafkaJavaSDK，代碼行數:23，代碼來源:KafkaConsumerTransducer.java

示例11: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public void run() {
	ConsumerIterator<String, String> it = stream.iterator();
	while (it.hasNext()) {
		MessageAndMetadata<String, String> consumerIterator = it.next();
		String uploadMessage = consumerIterator.message();
		System.out.println(Thread.currentThread().getName()
				+ " from partiton[" + consumerIterator.partition() + "]: "
				+ uploadMessage);
		try {
			sendDataToIotdb.writeData(uploadMessage); // upload data to the IoTDB database

		} catch (Exception ex) {
			System.out.println("SQLException: " + ex.getMessage());
		}
	}
}
開發者ID:thulab，項目名稱:iotdb-jdbc，代碼行數:17，代碼來源:KafkaConsumer.java

示例12: collectMq

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public void collectMq(){
	Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
       topicCountMap.put(Constants.kfTopic, new Integer(1));

       StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
       StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

       Map<String, List<KafkaStream<String, String>>> consumerMap =
               consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
       
       KafkaStream<String, String> stream = consumerMap.get(Constants.kfTopic).get(0);
       ConsumerIterator<String, String> it = stream.iterator();
       MessageAndMetadata<String, String> msgMeta;
       while (it.hasNext()){
       	msgMeta = it.next();
       	super.mqTimer.parseMqText(msgMeta.key(), msgMeta.message());
       	//System.out.println(msgMeta.key()+"\t"+msgMeta.message());
       }
}
開發者ID:lrtdc，項目名稱:light_drtc，代碼行數:20，代碼來源:KafkaMqCollect.java

示例13: consumeMessages

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
private void consumeMessages() {
    final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
    topicCountMap.put(TOPIC, 1);
    final StringDecoder decoder =
            new StringDecoder(new VerifiableProperties());
    final Map<String, List<KafkaStream<String, String>>> consumerMap =
            consumer.createMessageStreams(topicCountMap, decoder, decoder);
    final KafkaStream<String, String> stream =
            consumerMap.get(TOPIC).get(0);
    final ConsumerIterator<String, String> iterator = stream.iterator();

    Thread kafkaMessageReceiverThread = new Thread(
            () -> {
                while (iterator.hasNext()) {
                    String msg = iterator.next().message();
                    msg = msg == null ? "<null>" : msg;
                    System.out.println("got message: " + msg);
                    messagesReceived.add(msg);
                }
            },
            "kafkaMessageReceiverThread"
    );
    kafkaMessageReceiverThread.start();

}
開發者ID:hubrick，項目名稱:vertx-kafka-service，代碼行數:26，代碼來源:KafkaProducerServiceIntegrationTest.java

示例14: readTopicToList

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
/**
 * Read topic to list, only using Kafka code.
 */
private static List<MessageAndMetadata<byte[], byte[]>> readTopicToList(String topicName, ConsumerConfig config, final int stopAfter) {
	ConsumerConnector consumerConnector = Consumer.createJavaConsumerConnector(config);
	// we request only one stream per consumer instance. Kafka will make sure that each consumer group
	// will see each message only once.
	Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
	Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumerConnector.createMessageStreams(topicCountMap);
	if (streams.size() != 1) {
		throw new RuntimeException("Expected only one message stream but got "+streams.size());
	}
	List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
	if (kafkaStreams == null) {
		throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
	}
	if (kafkaStreams.size() != 1) {
		throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
	}
	LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, config.groupId());
	ConsumerIterator<byte[], byte[]> iteratorToRead = kafkaStreams.get(0).iterator();

	List<MessageAndMetadata<byte[], byte[]>> result = new ArrayList<>();
	int read = 0;
	while(iteratorToRead.hasNext()) {
		read++;
		result.add(iteratorToRead.next());
		if (read == stopAfter) {
			LOG.info("Read "+read+" elements");
			return result;
		}
	}
	return result;
}
開發者ID:axbaretto，項目名稱:flink，代碼行數:35，代碼來源:KafkaConsumerTestBase.java

示例15: run

import kafka.consumer.ConsumerIterator; //導入依賴的package包/類
public void run() {
	try {
		while(true){
			ConsumerIterator<byte[], byte[]> it = m_stream.iterator();
			while (it.hasNext()) {
				String m = null;
				try {
					m = new String(it.next().message(),
							this.kafkaInput.encoding);
					Map<String, Object> event = this.decoder
							.decode(m);
					if (event!=null&&event.size()>0){
						this.kafkaInput.process(event);
					} 
				} catch (Exception e) {
					logger.error("process event:{} failed:{}",m,e.getCause());
				}
			}
		}
	} catch (Exception t) {
		logger.error("kakfa Consumer fetch is error:{}",t.getCause());
	}
}
開發者ID:DTStack，項目名稱:jlogstash-input-plugin，代碼行數:24，代碼來源:Kafka.java


注：本文中的kafka.consumer.ConsumerIterator類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。