當前位置: 首頁>>代碼示例>>Java>>正文

Java SimpleConsumer類代碼示例

本文整理匯總了Java中kafka.javaapi.consumer.SimpleConsumer類的典型用法代碼示例。如果您正苦於以下問題：Java SimpleConsumer類的具體用法？Java SimpleConsumer怎麽用？Java SimpleConsumer使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


SimpleConsumer類屬於kafka.javaapi.consumer包，在下文中一共展示了SimpleConsumer類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: main

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
public static void main(String[] args) throws Exception {
	final String topic = "test2";
	String clientId = "LowLevelConsumerClient1";
	SimpleConsumer simpleConsumer = new SimpleConsumer(
			"192.168.1.186", 9092, 6000000, 64 * 1000000, clientId);
	FetchRequest req = new FetchRequestBuilder().clientId(clientId)
							.addFetch(topic, 0, 0L, 1000000)
							.addFetch(topic, 1, 0L, 1000000)
							.addFetch(topic, 2, 0L, 1000000)
							.build();
	FetchResponse rep = simpleConsumer.fetch(req);						
	ByteBufferMessageSet messageSet = rep.messageSet(topic, 0);
	for(MessageAndOffset messageAndOffset : messageSet) {
		ByteBuffer payload = messageAndOffset.message().payload();
		long offset = messageAndOffset.offset();
		byte[] bytes = new byte[payload.limit()];
		payload.get(bytes);
		System.out.println("Offset : " + offset + ", Payload : " + new String(bytes, "UTF-8"));
	}
}
開發者ID:walle-liao，項目名稱:jaf-examples，代碼行數:21，代碼來源:LowLevelConsumerDemo.java

示例2: fetchTopicMetadataFromBroker

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < NUM_TRIES_FETCH_TOPIC; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
開發者ID:Hanmourang，項目名稱:Gobblin，代碼行數:25，代碼來源:KafkaWrapper.java

示例3: getPartitionMetadata

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
public static PartitionMetadata getPartitionMetadata(final SimpleConsumer consumer, final List<String> topics, final int partitionId) {
    try {
        TopicMetadataRequest req = new TopicMetadataRequest(topics);
        TopicMetadataResponse resp = consumer.send(req);

        List<TopicMetadata> topicMetadataList = resp.topicsMetadata();

        for (TopicMetadata metaData : topicMetadataList) {
            for (PartitionMetadata part : metaData.partitionsMetadata()) {
                if (part.partitionId() == partitionId) {
                    return part;
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Unable to fetch partition meta data from host[{}:{}] [{}:{}]", consumer.host(), consumer.port(), topics, partitionId, e);
    }

    return null;
}
開發者ID:jeoffreylim，項目名稱:maelstrom，代碼行數:21，代碼來源:KafkaMetaData.java

示例4: main

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
public static void main(String[] args) throws Exception {
	final String topic = "topic1";
	String clientID = "DemoLowLevelConsumer1";
	SimpleConsumer simpleConsumer = new SimpleConsumer("kafka0", 9092, 100000, 64 * 1000000, clientID);
	FetchRequest req = new FetchRequestBuilder().clientId(clientID)
			.addFetch(topic, 0, 0L, 50).addFetch(topic, 1, 0L, 5000).addFetch(topic, 2, 0L, 1000000).build();
	FetchResponse fetchResponse = simpleConsumer.fetch(req);
	ByteBufferMessageSet messageSet = (ByteBufferMessageSet) fetchResponse.messageSet(topic, 0);
	for (MessageAndOffset messageAndOffset : messageSet) {
		ByteBuffer payload = messageAndOffset.message().payload();
		long offset = messageAndOffset.offset();
		byte[] bytes = new byte[payload.limit()];
		payload.get(bytes);
		System.out.println("Offset:" + offset + ", Payload:" + new String(bytes, "UTF-8"));
	}
}
開發者ID:habren，項目名稱:KafkaExample，代碼行數:17，代碼來源:DemoLowLevelConsumer.java

示例5: updateLeaderMap

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private void updateLeaderMap() {
  for (String broker : brokerList) {
    try {
      SimpleConsumer consumer = getSimpleConsumer(broker);
      TopicMetadataRequest req = new TopicMetadataRequest(auditTopics);
      kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
      List<TopicMetadata> metaData = resp.topicsMetadata();

      for (TopicMetadata tmd : metaData) {
        for (PartitionMetadata pmd : tmd.partitionsMetadata()) {
          TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId());
          partitionLeader.put(topicAndPartition, getHostPort(pmd.leader()));
        }
      }
    } catch (Exception e) {
      logger.warn("Got exception to get metadata from broker=" + broker, e);
    }
  }
}
開發者ID:uber，項目名稱:chaperone，代碼行數:20，代碼來源:KafkaMonitor.java

示例6: fetchLatestRecordPayloadBytes

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private byte[] fetchLatestRecordPayloadBytes(SimpleConsumer kafkaConsumer) {
  FetchRequest fetchRequest = new FetchRequestBuilder().addFetch(destinationTopic, 0, 0, 1000000).build();
  FetchResponse response = kafkaConsumer.fetch(fetchRequest);

  Iterator<MessageAndOffset> messageSetItr = response.messageSet(destinationTopic, 0).iterator();

  // Fast forward to the message at the latest offset in the topic
  MessageAndOffset latestMessage = new MessageAndOffset(new Message(new byte[] { }), 0L);
  while (messageSetItr.hasNext()) {
    latestMessage = messageSetItr.next();
  }

  ByteBuffer payload = latestMessage.message().payload();
  byte[] bytes = new byte[payload.limit()];
  payload.get(bytes);
  return bytes;
}
開發者ID:verisign，項目名稱:storm-graphite，代碼行數:18，代碼來源:BaseKafkaReporterTest.java

示例7: getOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private static OffsetInfo getOffset(String topic, PartitionMetadata partition) {
  Broker broker = partition.leader();

  SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 10000, 1000000,
                                               "com.rekko.newrelic.storm.kafka");
  try {
    TopicAndPartition
        topicAndPartition =
        new TopicAndPartition(topic, partition.partitionId());
    PartitionOffsetRequestInfo rquest = new PartitionOffsetRequestInfo(-1, 1);
    Map<TopicAndPartition, PartitionOffsetRequestInfo>
        map =
        new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    map.put(topicAndPartition, rquest);
    OffsetRequest req = new OffsetRequest(map, (short) 0, "com.rekko.newrelic.storm.kafka");
    OffsetResponse resp = consumer.getOffsetsBefore(req);
    OffsetInfo offset = new OffsetInfo();
    offset.offset = resp.offsets(topic, partition.partitionId())[0];
    return offset;
  } finally {
    consumer.close();
  }
}
開發者ID:ghais，項目名稱:newrelic_storm_kafka，代碼行數:24，代碼來源:Kafka.java

示例8: getLastOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String
        clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition,
            PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic,
                partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
開發者ID:wngn123，項目名稱:wngn-jms-kafka，代碼行數:17，代碼來源:SimpleConsumerExample.java

示例9: commitOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private boolean commitOffset(SimpleConsumer consumer, long offset, String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, OffsetAndMetadata> requestInfo = new HashMap<TopicAndPartition, OffsetAndMetadata>();
	OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(
			new OffsetMetadata(offset, OffsetMetadata.NoMetadata()), offset, -1L);
	requestInfo.put(topicAndPartition, offsetAndMetadata);
	OffsetCommitRequest commitRequest = new OffsetCommitRequest(groupid, requestInfo, correlationId, clientName,
			OffsetRequest.CurrentVersion());
	OffsetCommitResponse response = null;
	while (true) {
		try {
			logger.debug("partition {} commit offest", partition);
			response = consumer.commitOffsets(commitRequest);
			if (response != null)
				break;
		} catch (Exception e) {
			logger.error("some error occur when fetch messages", e);
			try {
				Thread.sleep(EXCEPTION_SLEEP_TIME);
			} catch (InterruptedException e1) {
				e1.printStackTrace();
			}
		}
	}
	return response.hasError();
}
開發者ID:onealcenter，項目名稱:ExactlyOncePartitionConsumer，代碼行數:27，代碼來源:ExactlyOncePartitionConsumer.java

示例10: getLastOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
開發者ID:warlock-china，項目名稱:azeroth，代碼行數:18，代碼來源:ZkConsumerCommand.java

示例11: getLastOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
/**
 * Defines where to start reading data from
 * Helpers Available:
 * kafka.api.OffsetRequest.EarliestTime() => finds the beginning of the data in the logs and starts streaming
 * from there
 * kafka.api.OffsetRequest.LatestTime()   => will only stream new messages
 *
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
開發者ID:bingoohuang，項目名稱:javacode-demo，代碼行數:30，代碼來源:SimpleExample.java

示例12: getLastOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
/***
     * Get the offset of the last message in the specified topic and partition
     *
     * @param consumer   consumer object
     * @param topic      topic name
     * @param partition  partition id
     * @param whichTime  time
     * @param clientName client name
     * @return offset value
     * @throws Exception if leader is not found or there is an error fetching data offset from broker.
     */
    public static long getLastOffset(kafka.javaapi.consumer.SimpleConsumer consumer, String topic, int partition,
                                     long whichTime, String clientName) throws Exception {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
        OffsetResponse response = consumer.getOffsetsBefore(request);

        if (response.hasError()) {
            throw new Exception("Error fetching data Offset from the Broker. Reason: " + response.errorCode(topic, partition));
//            return 0;
        }

        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
開發者ID:Microsoft，項目名稱:Availability-Monitor-for-Kafka，代碼行數:29，代碼來源:Consumer.java

示例13: getLastOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
開發者ID:vakinge，項目名稱:jeesuite-libs，代碼行數:17，代碼來源:ZkConsumerCommand.java

示例14: getLatestOffset

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request =
      new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}
開發者ID:uber，項目名稱:chaperone，代碼行數:17，代碼來源:KafkaMonitor.java

示例15: main

import kafka.javaapi.consumer.SimpleConsumer; //導入依賴的package包/類
public static void main(String... args) {


        KafkaService kafkaService = new KafkaServiceBuilder()
                .addBootstrapServer("localhost", 9092)
                .addZookeeperConnectServer("127.0.0.1", 2181).build();

        kafkaService.sendMessage("mom");

        SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.kafkaServerURL,
                KafkaProperties.kafkaServerPort,
                KafkaProperties.connectionTimeOut,
                KafkaProperties.kafkaProducerBufferSize,
                KafkaProperties.clientId);

//        FetchRequest req = new FetchRequestBuilder()
//                .clientId(KafkaProperties.clientId)
//                .addFetch(KafkaProperties.topic2, 0, 0L, 100)
//                .build();

        Sys.sleep(1000);
    }
開發者ID:advantageous，項目名稱:qbit-extensions，代碼行數:23，代碼來源:KafkaServiceBuilder.java


注：本文中的kafka.javaapi.consumer.SimpleConsumer類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。