Java ConsumerRecords.iterator方法代码示例

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
@Override
protected KeyMessage<K,V> computeNext() {
  if (iterator == null || !iterator.hasNext()) {
    try {
      long timeout = MIN_POLL_MS;
      ConsumerRecords<K, V> records;

      while ((records = consumer.poll(timeout)).isEmpty()) {
        timeout = Math.min(MAX_POLL_MS, timeout * 2);
      }
      iterator = records.iterator();
    } catch (Exception e) {
      consumer.close();
      return endOfData();
    }
  }
  ConsumerRecord<K,V> mm = iterator.next();
  return new KeyMessageImpl<>(mm.key(), mm.value());
}
 

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
private synchronized void pollAndDispatchMessage() throws InterruptedException {
	// 处理记录过程中，不能修改consumer相关的设定
	// 拉取需要处理的记录
	ConsumerRecords<String, byte[]> allRecords = consumer.poll(10000);

	// 为每个消息都封装成CALLABLE的形式，并进行调用处理
	Iterator<ConsumerRecord<String, byte[]>> iterator = allRecords.iterator();
	List<MessageHandler> listJob = new LinkedList<>();
	while (iterator.hasNext()) {
		listJob.add(new MessageHandler(iterator.next()));
	}
	executeJobs(listJob);
	// 全部调用成功，更新消费坐标
	consumer.commitAsync();
}
 

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
private List<KafkaResult> consume() {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords consumerRecords = kafkaConsumer.poll(clientConfig.getPollTimeoutMs());

    logger.info("Consumed {} records", consumerRecords.count());
    final Iterator<ConsumerRecord> recordIterator = consumerRecords.iterator();
    while (recordIterator.hasNext()) {
        // Get next record
        final ConsumerRecord consumerRecord = recordIterator.next();

        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit();
    return kafkaResultList;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
@Override
public String readLine() {
	String line = "";
	ConsumerRecords<String, String> records = this.consumer.poll(1000);
	Iterator<ConsumerRecord<String, String>> iterator = records.iterator();
	if (iterator.hasNext()) {
		ConsumerRecord<String, String> record = iterator.next();
		logger.info(String.format("offset = %d, key = %s, value = %s\n", record.offset(), record.key(), record.value()));
		line = record.value();
	}
	return line;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
@Override
public ConsumerRecords onConsume(final ConsumerRecords records) {

    final Map<TopicPartition, List<ConsumerRecord>> filteredRecords = new HashMap<>();

    // Iterate thru records
    final Iterator<ConsumerRecord> recordIterator = records.iterator();
    while (recordIterator.hasNext()) {
        final ConsumerRecord record = recordIterator.next();

        boolean result = true;

        // Iterate through filters
        for (final RecordFilterDefinition recordFilterDefinition : recordFilterDefinitions) {
            // Pass through filter
            result = recordFilterDefinition.getRecordFilter().includeRecord(
                record.topic(),
                record.partition(),
                record.offset(),
                record.key(),
                record.value()
            );

            // If we return false
            if (!result) {
                // break out of loop
                break;
            }
        }

        // If filter return true
        if (result) {
            // Include it in the results
            final TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());
            filteredRecords.putIfAbsent(topicPartition, new ArrayList<>());
            filteredRecords.get(topicPartition).add(record);
        }
    }

    // return filtered results
    return new ConsumerRecords(filteredRecords);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecords; //导入方法依赖的package包/类
/**
 * Test that filters can filter messages.
 */
@Test
public void testFilterMessages() {
    final int totalRecords = 5;

    // Create mock Filters
    final RecordFilter mockFilter1 = mock(RecordFilter.class);
    final RecordFilter mockFilter2 = mock(RecordFilter.class);

    when(mockFilter1.includeRecord(eq("MyTopic"), eq(0), anyLong(), anyObject(), anyObject()))
        .thenReturn(true, false, true, true, true);
    when(mockFilter2.includeRecord(eq("MyTopic"), eq(0), anyLong(), anyObject(), anyObject()))
        .thenReturn(true, true, false, true);

    final RecordFilterDefinition recordFilterDefinition1 = new RecordFilterDefinition(mockFilter1, new HashMap<>());
    final RecordFilterDefinition recordFilterDefinition2 = new RecordFilterDefinition(mockFilter2, new HashMap<>());

    // Create ConsumerConfigs
    final Map<String, Object> consumerConfigs = new HashMap<>();
    consumerConfigs.put(RecordFilterInterceptor.CONFIG_KEY, Lists.newArrayList(recordFilterDefinition1, recordFilterDefinition2));

    // Create interceptor.
    final RecordFilterInterceptor interceptor = new RecordFilterInterceptor();

    // Call configure
    interceptor.configure(consumerConfigs);

    // Create ConsumerRecords
    final ConsumerRecords consumerRecords = createConsumerRecords(totalRecords);

    // Pass through interceptor
    final ConsumerRecords results = interceptor.onConsume(consumerRecords);

    // Validate we got the expected results
    assertEquals("Should have 3 records", totalRecords - 2, results.count());

    for (Iterator<ConsumerRecord> it = results.iterator(); it.hasNext(); ) {
        final ConsumerRecord consumerRecord = it.next();
        assertNotEquals("Should not have offsets 1 and 3", 1, consumerRecord.offset());
        assertNotEquals("Should not have offsets 1 and 3", 3, consumerRecord.offset());
    }

    // Verify mocks
    verify(mockFilter1, times(totalRecords))
        .includeRecord(eq("MyTopic"), eq(0), anyLong(), anyObject(), anyObject());
    verify(mockFilter2, times(totalRecords - 1))
        .includeRecord(eq("MyTopic"), eq(0), anyLong(), anyObject(), anyObject());
}