Java SimpleConsumer.close方法代碼示例

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < NUM_TRIES_FETCH_TOPIC; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
private static OffsetInfo getOffset(String topic, PartitionMetadata partition) {
  Broker broker = partition.leader();

  SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 10000, 1000000,
                                               "com.rekko.newrelic.storm.kafka");
  try {
    TopicAndPartition
        topicAndPartition =
        new TopicAndPartition(topic, partition.partitionId());
    PartitionOffsetRequestInfo rquest = new PartitionOffsetRequestInfo(-1, 1);
    Map<TopicAndPartition, PartitionOffsetRequestInfo>
        map =
        new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    map.put(topicAndPartition, rquest);
    OffsetRequest req = new OffsetRequest(map, (short) 0, "com.rekko.newrelic.storm.kafka");
    OffsetResponse resp = consumer.getOffsetsBefore(req);
    OffsetInfo offset = new OffsetInfo();
    offset.offset = resp.offsets(topic, partition.partitionId())[0];
    return offset;
  } finally {
    consumer.close();
  }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
private List<TopicMetadata> getTopicMetadataFromBroker(List<KafkaNode> bootstrapNodes, String topicName) throws NoBrokerAvailableException {
    Exception lastException = null;
    for (KafkaNode bootstrapNode : bootstrapNodes) {
        SimpleConsumer consumer = null;

        try {
            consumer = createConsumer(bootstrapNode);
            final TopicMetadataRequest request = new TopicMetadataRequest(Collections.singletonList(topicName));
            final TopicMetadataResponse response = consumer.send(request);

            return response.topicsMetadata();
        } catch (Exception e) {
            lastException = e;
        } finally {
            if (consumer != null) {
                consumer.close();
            }
        }
    }

    final String message = String.format("No broker available for topic '%s' with servers '%s'", topicName, Arrays.toString(bootstrapNodes.toArray()));
    throw new NoBrokerAvailableException(message, lastException);
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
@Override
public void close() throws IOException {
  int numOfConsumersNotClosed = 0;

  for (SimpleConsumer consumer : this.activeConsumers.values()) {
    if (consumer != null) {
      try {
        consumer.close();
      } catch (Exception e) {
        LOG.warn(String.format("Failed to close Kafka Consumer %s:%d", consumer.host(), consumer.port()));
        numOfConsumersNotClosed++;
      }
    }
  }
  this.activeConsumers.clear();
  if (numOfConsumersNotClosed > 0) {
    throw new IOException(numOfConsumersNotClosed + " consumer(s) failed to close.");
  }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
public List<byte[]> readMessages(String topic) {
  SimpleConsumer consumer = new SimpleConsumer("localhost", 6667, 100000, 64 * 1024, "consumer");
  FetchRequest req = new FetchRequestBuilder()
          .clientId("consumer")
          .addFetch(topic, 0, 0, 100000)
          .build();
  FetchResponse fetchResponse = consumer.fetch(req);
  Iterator<MessageAndOffset> results = fetchResponse.messageSet(topic, 0).iterator();
  List<byte[]> messages = new ArrayList<>();
  while(results.hasNext()) {
    ByteBuffer payload = results.next().message().payload();
    byte[] bytes = new byte[payload.limit()];
    payload.get(bytes);
    messages.add(bytes);
  }
  consumer.close();
  return messages;
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
@Override
public long getEarliestOffset() {
  if (this.earliestOffset == -2 && uri != null) {
    // TODO : Make the hardcoded paramters configurable
    SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
        1024 * 1024, "hadoop-etl");
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
        kafka.api.OffsetRequest.EarliestTime(), 1));
    OffsetResponse response = consumer
        .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
            .CurrentVersion(), "hadoop-etl"));
    long[] endOffset = response.offsets(topic, partition);
    consumer.close();
    this.earliestOffset = endOffset[0];
    return endOffset[0];
  } else {
    return this.earliestOffset;
  }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
@Override
public long getLastOffset(long time) {
  SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
      1024 * 1024, "hadoop-etl");
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
      time, 1));
  OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
      kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
  long[] endOffset = response.offsets(topic, partition);
  consumer.close();
  if(endOffset.length == 0)
  {
    log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition);
  }
  this.latestOffset = endOffset[0];
  return endOffset[0];
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  log.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
@Override
public void close() throws IOException {
  int numOfConsumersNotClosed = 0;

  for (SimpleConsumer consumer : this.activeConsumers.values()) {
    if (consumer != null) {
      try {
        consumer.close();
      } catch (Exception e) {
        log.warn(String.format("Failed to close Kafka Consumer %s:%d", consumer.host(), consumer.port()));
        numOfConsumersNotClosed++;
      }
    }
  }
  this.activeConsumers.clear();
  if (numOfConsumersNotClosed > 0) {
    throw new IOException(numOfConsumersNotClosed + " consumer(s) failed to close.");
  }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
public int getNumPartitions(String topic) {
    SimpleConsumer consumer = null;
    try {
        consumer = createConsumer(
            mConfig.getKafkaSeedBrokerHost(),
            mConfig.getKafkaSeedBrokerPort(),
            "partitionLookup");
        List<String> topics = new ArrayList<String>();
        topics.add(topic);
        TopicMetadataRequest request = new TopicMetadataRequest(topics);
        TopicMetadataResponse response = consumer.send(request);
        if (response.topicsMetadata().size() != 1) {
            throw new RuntimeException("Expected one metadata for topic " + topic + " found " +
                response.topicsMetadata().size());
        }
        TopicMetadata topicMetadata = response.topicsMetadata().get(0);
        return topicMetadata.partitionsMetadata().size();
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
public Message getCommittedMessage(TopicPartition topicPartition) throws Exception {
    SimpleConsumer consumer = null;
    try {
        long committedOffset = mZookeeperConnector.getCommittedOffsetCount(topicPartition) - 1;
        if (committedOffset < 0) {
            return null;
        }
        consumer = createConsumer(topicPartition);
        return getMessage(topicPartition, committedOffset, consumer);
    } catch (MessageDoesNotExistException e) {
      // If a RuntimeEMessageDoesNotExistException exception is raised,
      // the message at the last comitted offset does not exist in Kafka.
      // This is usually due to the message being compacted away by the
      // Kafka log compaction process.
      //
      // That is no an exceptional situation - in fact it can be normal if
      // the topic being consumed by Secor has a low volume. So in that
      // case, simply return null
      LOG.warn("no committed message for topic {} partition {}", topicPartition.getTopic(), topicPartition.getPartition());
      return null;
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
/**
 * 關閉對應資源
 *
 * @param consumer
 */
private static void closeSimpleConsumer(SimpleConsumer consumer) {
    if (consumer != null) {
        try {
            consumer.close();
        } catch (Exception e) {
        }
    }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
/**
 * 獲取指定主題及分區logsize
 * @param stat
 */
public void getTopicPartitionLogSize(TopicPartitionInfo stat) {
    BrokerEndPoint leader = findLeader(stat.getTopic(), stat.getPartition()).leader();
    SimpleConsumer consumer = getConsumerClient(leader.host(), leader.port());

    try {
        long logsize = getLastOffset(consumer, stat.getTopic(), stat.getPartition(),
            kafka.api.OffsetRequest.LatestTime());
        stat.setLogSize(logsize);
    } finally {
        consumer.close();
    }
}
 

import kafka.javaapi.consumer.SimpleConsumer; //導入方法依賴的package包/類
public static long getLastOffset(String leadBrokerHost, int port, String topic, 
        int partition, long whichTime) {
    String clientName = "Client_" + topic + "_" + partition;
    SimpleConsumer consumer = new SimpleConsumer(leadBrokerHost, port, 
            soTimeoutMS, bufferSize, clientName);
    
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = 
            new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
            kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        LOG.warn("Error fetching data Offset Data the Broker. Reason: " 
                + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    
    if (consumer != null) {
        try {
            consumer.close();
        } catch (Exception e) {
            LOG.error("SimpleConsumer.close() error due to ", e);
        }
    }
    
    return offsets.length > 0 ? offsets[0] : -1;
}