本文整理汇总了Java中kafka.consumer.ConsumerIterator类的典型用法代码示例。如果您正苦于以下问题:Java ConsumerIterator类的具体用法?Java ConsumerIterator怎么用?Java ConsumerIterator使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
ConsumerIterator类属于kafka.consumer包,在下文中一共展示了ConsumerIterator类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
/**
* When an object implementing interface <code>Runnable</code> is used
* to create a thread, starting the thread causes the object's
* <code>run</code> method to be called in that separately executing
* thread.
* <p>
* The general contract of the method <code>run</code> is that it may
* take any action whatsoever.
*
* @see Thread#run()
*/
@Override
public void run() {
ConsumerConnector consumerConnector = KafkaUtils.createConsumerConnector(zkAddr, group);
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(CONSUMER_OFFSET_TOPIC, new Integer(1));
KafkaStream<byte[], byte[]> offsetMsgStream = consumerConnector.createMessageStreams(topicCountMap).get(CONSUMER_OFFSET_TOPIC).get(0);
ConsumerIterator<byte[], byte[]> it = offsetMsgStream.iterator();
while (true) {
MessageAndMetadata<byte[], byte[]> offsetMsg = it.next();
if (ByteBuffer.wrap(offsetMsg.key()).getShort() < 2) {
try {
GroupTopicPartition commitKey = readMessageKey(ByteBuffer.wrap(offsetMsg.key()));
if (offsetMsg.message() == null) {
continue;
}
kafka.common.OffsetAndMetadata commitValue = readMessageValue(ByteBuffer.wrap(offsetMsg.message()));
kafkaConsumerOffsets.put(commitKey, commitValue);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
示例2: getNextMessage
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public MessageAndMetadata getNextMessage(String topic) {
List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);
// it has only a single stream, because there is only one consumer
KafkaStream stream = streams.get(0);
final ConsumerIterator<byte[], byte[]> it = stream.iterator();
int counter = 0;
try {
if (it.hasNext()) {
return it.next();
} else {
return null;
}
} catch (ConsumerTimeoutException e) {
logger.error("0 messages available to fetch for the topic " + topic);
return null;
}
}
示例3: nextTuple
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public void nextTuple() {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(TopologyConfig.kafkaTopic, 1);//one excutor - one thread
Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = conn.createMessageStreams(topicCountMap);
List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(kafkaTopic);
ConsumerIterator<byte[], byte[]> iter = streams.get(0).iterator();
while(true){
while(iter.hasNext()){
String s = new String(iter.next().message());
collector.emit(new Values(s));
UUID msgId = UUID.randomUUID();
this.pending.put(msgId, new Values(s));
}
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
logger.error("Spout : sleep wrong \n", e);
}
}
}
示例4: processStreamsByTopic
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
private void processStreamsByTopic(String topicKeys, List<KafkaStream<byte[], byte[]>> streamList) {
// init stream thread pool
ExecutorService streamPool = Executors.newFixedThreadPool(partitions);
String[] topics = StringUtils.split(topicKeys, ",");
if (log.isDebugEnabled())
log.debug("准备处理消息流集合 KafkaStreamList,topic count={},topics={}, partitions/topic={}", topics.length, topicKeys, partitions);
//遍历stream
AtomicInteger index = new AtomicInteger(0);
for (KafkaStream<byte[], byte[]> stream : streamList) {
Thread streamThread = new Thread() {
@Override
public void run() {
int i = index.getAndAdd(1);
if (log.isDebugEnabled())
log.debug("处理消息流KafkaStream -- No.={}, partitions={}", i, partitions + ":" + i);
ConsumerIterator<byte[], byte[]> consumerIterator = stream.iterator();
processStreamByConsumer(topicKeys, consumerIterator);
}
};
streamPool.execute(streamThread);
}
}
示例5: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public void run() {
try {
while(true){
ConsumerIterator<byte[], byte[]> it = m_stream.iterator();
while (it.hasNext()) {
String m = null;
try {
m = new String(it.next().message(),
this.kafkaInput.encoding);
Map<String, Object> event = this.decoder
.decode(m);
if(zkDistributed==null){
this.kafkaInput.process(event);
}else{
zkDistributed.route(event);
}
} catch (Exception e) {
logger.error("process event:{} failed:{}",m,ExceptionUtil.getErrorMessage(e));
}
}
}
} catch (Exception t) {
logger.error("kakfa Consumer fetch is error:{}",ExceptionUtil.getErrorMessage(t));
}
}
示例6: shouldWriteThenRead
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
@Test
public void shouldWriteThenRead() throws Exception {
//Create a consumer
ConsumerIterator<String, String> it = buildConsumer(Original.topic);
//Create a producer
producer = new KafkaProducer<>(producerProps());
//send a message
producer.send(new ProducerRecord<>(Original.topic, "message")).get();
//read it back
MessageAndMetadata<String, String> messageAndMetadata = it.next();
String value = messageAndMetadata.message();
assertThat(value, is("message"));
}
示例7: shouldWriteThenRead
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
@Test
public void shouldWriteThenRead() throws Exception {
//Create a consumer
ConsumerIterator<String, String> it = buildConsumer(SimpleKafkaTest.topic);
//Create a producer
producer = new KafkaProducer<>(producerProps());
//send a message
producer.send(new ProducerRecord<>(SimpleKafkaTest.topic, "message")).get();
//read it back
MessageAndMetadata<String, String> messageAndMetadata = it.next();
String value = messageAndMetadata.message();
assertThat(value, is("message"));
}
示例8: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
@Override
public void run() {
ConsumerIterator<byte[], byte[]> it = stream.iterator();
while (it.hasNext()) {
MessageAndMetadata<byte[], byte[]> mam = it.next();
String jsonStr = "";
try {
jsonStr = new String(mam.message());
JSONObject jsonObject = JSONObject.parseObject(jsonStr);
LogcenterConfig config = LogConfigCache.getLogConfigCache(jsonObject);
IStorageApi iStorageApi = ServiceRegister.getInstance().getProvider(config.getStorageType());
iStorageApi.save(jsonObject);
} catch (Exception e) {
e.printStackTrace();
logger.error("partition[" + mam.partition() + "]," + "offset[" + mam.offset() + "], " + jsonStr, e);
continue;
}
}
}
示例9: testLogging
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
@Test
public void testLogging() throws InterruptedException {
for (int i = 0; i<1000; ++i) {
logger.info("message"+i);
}
final KafkaStream<byte[], byte[]> log = kafka.createClient().createMessageStreamsByFilter(new Whitelist("logs"),1).get(0);
final ConsumerIterator<byte[], byte[]> iterator = log.iterator();
for (int i=0; i<1000; ++i) {
final String messageFromKafka = new String(iterator.next().message(), UTF8);
assertThat(messageFromKafka, Matchers.equalTo("message"+i));
}
}
示例10: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
@Override
public void run() {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(transducer_topic, new Integer(1));
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
Map<String, List<KafkaStream<String, String>>> consumerMap =
consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
KafkaStream<String, String> stream = consumerMap.get(transducer_topic).get(0);
ConsumerIterator<String, String> it = stream.iterator();
while (it.hasNext() && bStartConsume){
transducerDataProcessor.newData(it.next().message());
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
示例11: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public void run() {
ConsumerIterator<String, String> it = stream.iterator();
while (it.hasNext()) {
MessageAndMetadata<String, String> consumerIterator = it.next();
String uploadMessage = consumerIterator.message();
System.out.println(Thread.currentThread().getName()
+ " from partiton[" + consumerIterator.partition() + "]: "
+ uploadMessage);
try {
sendDataToIotdb.writeData(uploadMessage); // upload data to the IoTDB database
} catch (Exception ex) {
System.out.println("SQLException: " + ex.getMessage());
}
}
}
示例12: collectMq
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public void collectMq(){
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(Constants.kfTopic, new Integer(1));
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
Map<String, List<KafkaStream<String, String>>> consumerMap =
consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
KafkaStream<String, String> stream = consumerMap.get(Constants.kfTopic).get(0);
ConsumerIterator<String, String> it = stream.iterator();
MessageAndMetadata<String, String> msgMeta;
while (it.hasNext()){
msgMeta = it.next();
super.mqTimer.parseMqText(msgMeta.key(), msgMeta.message());
//System.out.println(msgMeta.key()+"\t"+msgMeta.message());
}
}
示例13: consumeMessages
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
private void consumeMessages() {
final Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(TOPIC, 1);
final StringDecoder decoder =
new StringDecoder(new VerifiableProperties());
final Map<String, List<KafkaStream<String, String>>> consumerMap =
consumer.createMessageStreams(topicCountMap, decoder, decoder);
final KafkaStream<String, String> stream =
consumerMap.get(TOPIC).get(0);
final ConsumerIterator<String, String> iterator = stream.iterator();
Thread kafkaMessageReceiverThread = new Thread(
() -> {
while (iterator.hasNext()) {
String msg = iterator.next().message();
msg = msg == null ? "<null>" : msg;
System.out.println("got message: " + msg);
messagesReceived.add(msg);
}
},
"kafkaMessageReceiverThread"
);
kafkaMessageReceiverThread.start();
}
示例14: readTopicToList
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
/**
* Read topic to list, only using Kafka code.
*/
private static List<MessageAndMetadata<byte[], byte[]>> readTopicToList(String topicName, ConsumerConfig config, final int stopAfter) {
ConsumerConnector consumerConnector = Consumer.createJavaConsumerConnector(config);
// we request only one stream per consumer instance. Kafka will make sure that each consumer group
// will see each message only once.
Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumerConnector.createMessageStreams(topicCountMap);
if (streams.size() != 1) {
throw new RuntimeException("Expected only one message stream but got "+streams.size());
}
List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
if (kafkaStreams == null) {
throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
}
if (kafkaStreams.size() != 1) {
throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
}
LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, config.groupId());
ConsumerIterator<byte[], byte[]> iteratorToRead = kafkaStreams.get(0).iterator();
List<MessageAndMetadata<byte[], byte[]>> result = new ArrayList<>();
int read = 0;
while(iteratorToRead.hasNext()) {
read++;
result.add(iteratorToRead.next());
if (read == stopAfter) {
LOG.info("Read "+read+" elements");
return result;
}
}
return result;
}
示例15: run
import kafka.consumer.ConsumerIterator; //导入依赖的package包/类
public void run() {
try {
while(true){
ConsumerIterator<byte[], byte[]> it = m_stream.iterator();
while (it.hasNext()) {
String m = null;
try {
m = new String(it.next().message(),
this.kafkaInput.encoding);
Map<String, Object> event = this.decoder
.decode(m);
if (event!=null&&event.size()>0){
this.kafkaInput.process(event);
}
} catch (Exception e) {
logger.error("process event:{} failed:{}",m,e.getCause());
}
}
}
} catch (Exception t) {
logger.error("kakfa Consumer fetch is error:{}",t.getCause());
}
}