Java ConsumerRecord.timestamp方法代碼示例

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
private List<KafkaResult> consume() {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords consumerRecords = kafkaConsumer.poll(clientConfig.getPollTimeoutMs());

    logger.info("Consumed {} records", consumerRecords.count());
    final Iterator<ConsumerRecord> recordIterator = consumerRecords.iterator();
    while (recordIterator.hasNext()) {
        // Get next record
        final ConsumerRecord consumerRecord = recordIterator.next();

        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit();
    return kafkaResultList;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
private void processSingleRecord(List<Long> txIds, ConsumerRecord<ByteBuffer, ByteBuffer> record) {
    long txId = record.timestamp();
    boolean found = txIds.remove(txId);
    if (found) {
        ProducerRecord<ByteBuffer, ByteBuffer> producerRecord =
                new ProducerRecord<>(clusterConfig.getGapTopic(), record.key(), record.value());
        producer.send(producerRecord);
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
public void start() {
    running = true;

    Runnable onStop = new Runnable() {
        @Override
        public void run() {
            //todo add logic
        }
    };

    try (Consumer<ByteBuffer, ByteBuffer> consumer = kafkaFactory.consumer(localConsumerProperties, onStop);
         Producer<ByteBuffer, ByteBuffer> producer = kafkaFactory.producer(replicaProducerProperties)) {
        int partitions = producer.partitionsFor(reconciliationTopic).size();

        consumer.subscribe(Collections.singletonList(localTopic));
        while (running) {
            ConsumerRecords<ByteBuffer, ByteBuffer> records = consumer.poll(POLL_TIMEOUT);

            for (ConsumerRecord<ByteBuffer, ByteBuffer> record : records) {
                long transactionId = record.timestamp();
                int partition = TransactionMessageUtil.partitionFor(transactionId, partitions);

                producer.send(new ProducerRecord<>(reconciliationTopic, partition, record.key(), record.value()));
            }
        }
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
@SuppressWarnings("unchecked")
@Override
public void update(final ConsumerRecord<byte[], byte[]> record) {
    final SourceNodeAndDeserializer sourceNodeAndDeserializer = deserializers.get(record.topic());
    final ConsumerRecord<Object, Object> deserialized = sourceNodeAndDeserializer.deserializer.deserialize(record);
    final ProcessorRecordContext recordContext =
            new ProcessorRecordContext(deserialized.timestamp(),
                                       deserialized.offset(),
                                       deserialized.partition(),
                                       deserialized.topic());
    processorContext.setRecordContext(recordContext);
    processorContext.setCurrentNode(sourceNodeAndDeserializer.sourceNode);
    sourceNodeAndDeserializer.sourceNode.process(deserialized.key(), deserialized.value());
    offsets.put(new TopicPartition(record.topic(), record.partition()), deserialized.offset() + 1);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
/**
 * Extracts the embedded metadata timestamp from the given {@link ConsumerRecord}.
 *
 * @param record a data record
 * @param previousTimestamp the latest extracted valid timestamp of the current record's partition˙ (could be -1 if unknown)
 * @return the embedded metadata timestamp of the given {@link ConsumerRecord}
 */
@Override
public long extract(final ConsumerRecord<Object, Object> record, final long previousTimestamp) {
    final long timestamp = record.timestamp();

    if (timestamp < 0) {
        return onInvalidTimestamp(record, timestamp, previousTimestamp);
    }

    return timestamp;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
@Override
public long extract(final ConsumerRecord<Object, Object> record, final long previousTimestamp) {
    if (record.value().toString().matches(".*@[0-9]+"))
        return Long.parseLong(record.value().toString().split("@")[1]);

    if (record.timestamp() > 0L)
        return record.timestamp();

    return timestamp;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
@Override
public void run() {
    // Rename thread.
    Thread.currentThread().setName("WebSocket Consumer: " + clientConfig.getConsumerId());
    logger.info("Starting socket consumer for {}", clientConfig.getConsumerId());

    // Determine where to start from.
    initializeStartingPosition(clientConfig.getStartingPosition());

    do {
        // Start trying to consume messages from kafka
        final ConsumerRecords consumerRecords = kafkaConsumer.poll(POLL_TIMEOUT_MS);

        // If no records found
        if (consumerRecords.isEmpty()) {
            // Sleep for a bit
            sleep(POLL_TIMEOUT_MS);

            // Skip to next iteration of loop.
            continue;
        }

        // Push messages onto output queue
        for (final ConsumerRecord consumerRecord : (Iterable<ConsumerRecord>) consumerRecords) {
            // Translate record
            final KafkaResult kafkaResult = new KafkaResult(
                consumerRecord.partition(),
                consumerRecord.offset(),
                consumerRecord.timestamp(),
                consumerRecord.key(),
                consumerRecord.value()
            );

            // Add to the queue, this operation may block, effectively preventing the consumer from
            // consuming unbounded-ly.
            try {
                outputQueue.put(kafkaResult);
            } catch (final InterruptedException interruptedException) {
                // InterruptedException means we should shut down.
                requestStop();
            }
        }

        // Sleep for a bit
        sleep(DWELL_TIME_MS);
    }
    while (!requestStop);

    // requestStop
    kafkaConsumer.close();

    logger.info("Shutdown consumer {}", clientConfig.getConsumerId());
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
@Override
public long extract(ConsumerRecord<Object, Object> consumerRecord, long l) {
    return consumerRecord.timestamp();
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //導入方法依賴的package包/類
@Override
public long extract(ConsumerRecord<Object, Object> record, long previousTimestamp) {

    if (record.value() instanceof Integer) return Long.valueOf(String.valueOf(record.value()));
    return record.timestamp();
}