Java ConsumerRecord.offset方法代码示例

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public List<StatMessage> poll() {
                /* 快速取，如果没有就立刻返回 */
    ConsumerRecords<String, String> records = consumer.poll(1000);
    if (records.count() == 0) {
        count++;
        if (count % 60 == 0) {
            count = 0;
            LOG.info(String.format("running on %s (offset=%d).......", statTopic,  consumer.position(statTopicPartition)));
        }
        return null;
    }

    LOG.info(String.format("KafkaSource got %d records......", records.count()));

    List<StatMessage> list = new ArrayList<>();
    for (ConsumerRecord<String, String> record : records) {
        String key = record.key();
        long offset = record.offset();

        StatMessage msg = StatMessage.parse(record.value());
        list.add(msg);
        //logger.info(String.format("KafkaSource got record key=%s, offset=%d......", key, offset));
    }

    return list;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@Override
public TransactionInput pollNextTransaction(long timeout, TimeUnit unit) {
    Iterator<ConsumerRecord<Integer, byte[]>> iterator = m_consumer.poll(unit.toMillis(timeout)).iterator();
    if (!iterator.hasNext()) {
        return null;
    }

    ConsumerRecord<Integer, byte[]> firstRecord = iterator.next();
    if (iterator.hasNext()) {
        throw new IllegalStateException("Kafka should not return more than one record");
    }

    return new TransactionInput() {
        @Override
        public InputStream getInputStream() throws IOException {
            return new ByteArrayInputStream(firstRecord.value());
        }

        @Override
        public long getTransactionId() {
            return firstRecord.offset();
        }
    };
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
static Message<? extends com.google.protobuf.Message> fromKafka(com.google.protobuf.Message protoMessage, Envelope envelope, ConsumerRecord<String, byte[]> record) {
    boolean wasReceived = true;

    Topic topic = new Topic(record.topic());
    String partitioningKey = record.key();
    int partitionId = record.partition();
    long offset = record.offset();

    String messageId = envelope.getMessageId();
    String correlationId = envelope.getCorrelationId();

    MessageType type = MessageType.of(protoMessage);

    String requestCorrelationId = envelope.getRequestCorrelationId();
    Topic replyTo = new Topic(envelope.getReplyTo());

    Metadata meta = new Metadata(wasReceived, topic, partitioningKey, partitionId, offset, messageId, correlationId, requestCorrelationId, replyTo, type);
    return new Message<>(protoMessage, meta);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private void convertMessages(ConsumerRecords<byte[], byte[]> msgs) {
    for (ConsumerRecord<byte[], byte[]> msg : msgs) {
        log.trace("Consuming message with key {}, value {}", msg.key(), msg.value());
        SchemaAndValue keyAndSchema = keyConverter.toConnectData(msg.topic(), msg.key());
        SchemaAndValue valueAndSchema = valueConverter.toConnectData(msg.topic(), msg.value());
        SinkRecord record = new SinkRecord(msg.topic(), msg.partition(),
                keyAndSchema.schema(), keyAndSchema.value(),
                valueAndSchema.schema(), valueAndSchema.value(),
                msg.offset(),
                ConnectUtils.checkAndConvertTimestamp(msg.timestamp()),
                msg.timestampType());
        record = transformationChain.apply(record);
        if (record != null) {
            messageBatch.add(record);
        }
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public static <K, V> Payload<K, V> decodePayload(Deserializer<V> valueDeserializer, ConsumerRecord<K, byte[]> originConsumerRecord) {
    TracingHeader tracingHeader = null;
    ConsumerRecord<K, V> dataRecord = null;
    boolean sampled = false;
    byte[] data = originConsumerRecord.value();
    byte[] vData = null;
    if (data.length <= HEADER_LENGTH) {
        vData = data;
    } else {
        ByteBuffer byteBuf = ByteBuffer.wrap(data);
        short magic = byteBuf.getShort(0);
        short tpLen = byteBuf.getShort(2);
        if (magic == MAGIC && tpLen == TracingHeader.LENGTH) {
            byte[] tpBytes = new byte[tpLen];
            System.arraycopy(byteBuf.array(), HEADER_LENGTH, tpBytes, 0, tpLen);
            tracingHeader = TracingHeader.fromBytes(tpBytes);
            sampled = true;
            int dataOffset = tpLen + HEADER_LENGTH;
            vData = new byte[byteBuf.array().length - dataOffset];
            System.arraycopy(byteBuf.array(), dataOffset, vData, 0, vData.length);
        } else {
            vData = data;
        }
    }
    dataRecord = new ConsumerRecord<>(originConsumerRecord.topic(),
            originConsumerRecord.partition(), originConsumerRecord.offset(),
            originConsumerRecord.key(), valueDeserializer.deserialize(originConsumerRecord.topic(), vData));
    return new Payload<>(tracingHeader, dataRecord, sampled);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private List<KafkaResult> consume() {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords consumerRecords = kafkaConsumer.poll(clientConfig.getPollTimeoutMs());

    logger.info("Consumed {} records", consumerRecords.count());
    final Iterator<ConsumerRecord> recordIterator = consumerRecords.iterator();
    while (recordIterator.hasNext()) {
        // Get next record
        final ConsumerRecord consumerRecord = recordIterator.next();

        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit();
    return kafkaResultList;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public TransactionWrapper(ConsumerRecord<ByteBuffer, ByteBuffer> record, TransactionMetadata deserializedMetadata) {
    GridArgumentCheck.notNull(deserializedMetadata, "metadata cannot be null");
    this.value = record.value();
    this.key = record.key();
    this.topicPartition = new TopicPartition(record.topic(), record.partition());
    this.offset = record.offset();
    this.deserializedMetadata = deserializedMetadata;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void update(final ConsumerRecord<byte[], byte[]> record) {
    final SourceNodeAndDeserializer sourceNodeAndDeserializer = deserializers.get(record.topic());
    final ConsumerRecord<Object, Object> deserialized = sourceNodeAndDeserializer.deserializer.deserialize(record);
    final ProcessorRecordContext recordContext =
            new ProcessorRecordContext(deserialized.timestamp(),
                                       deserialized.offset(),
                                       deserialized.partition(),
                                       deserialized.topic());
    processorContext.setRecordContext(recordContext);
    processorContext.setCurrentNode(sourceNodeAndDeserializer.sourceNode);
    sourceNodeAndDeserializer.sourceNode.process(deserialized.key(), deserialized.value());
    offsets.put(new TopicPartition(record.topic(), record.partition()), deserialized.offset() + 1);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private void restoreState(final StateRestoreCallback stateRestoreCallback,
                          final List<TopicPartition> topicPartitions,
                          final Map<TopicPartition, Long> highWatermarks) {
    for (final TopicPartition topicPartition : topicPartitions) {
        consumer.assign(Collections.singletonList(topicPartition));
        final Long checkpoint = checkpointableOffsets.get(topicPartition);
        if (checkpoint != null) {
            consumer.seek(topicPartition, checkpoint);
        } else {
            consumer.seekToBeginning(Collections.singletonList(topicPartition));
        }

        long offset = consumer.position(topicPartition);
        final Long highWatermark = highWatermarks.get(topicPartition);

        while (offset < highWatermark) {
            final ConsumerRecords<byte[], byte[]> records = consumer.poll(100);
            for (ConsumerRecord<byte[], byte[]> record : records) {
                offset = record.offset() + 1;
                if (record.key() != null) {
                    stateRestoreCallback.restore(record.key(), record.value());
                }
            }
        }
        checkpointableOffsets.put(topicPartition, offset);
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private long processNext(final List<ConsumerRecord<byte[], byte[]>> records, final StateRestorer restorer, final Long endOffset) {
    for (final ConsumerRecord<byte[], byte[]> record : records) {
        final long offset = record.offset();
        if (restorer.hasCompleted(offset, endOffset)) {
            return offset;
        }
        if (record.key() != null) {
            restorer.restore(record.key(), record.value());
        }
    }
    return consumer.position(restorer.partition());
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
List<ConsumerRecord<byte[], byte[]>> updateStandbyStates(final TopicPartition storePartition,
                                                         final List<ConsumerRecord<byte[], byte[]>> records) {
    final long limit = offsetLimit(storePartition);
    List<ConsumerRecord<byte[], byte[]>> remainingRecords = null;

    // restore states from changelog records
    final StateRestoreCallback restoreCallback = restoreCallbacks.get(storePartition.topic());

    long lastOffset = -1L;
    int count = 0;
    for (final ConsumerRecord<byte[], byte[]> record : records) {
        if (record.offset() < limit) {
            try {
                restoreCallback.restore(record.key(), record.value());
            } catch (final Exception e) {
                throw new ProcessorStateException(String.format("%s exception caught while trying to restore state from %s", logPrefix, storePartition), e);
            }
            lastOffset = record.offset();
        } else {
            if (remainingRecords == null) {
                remainingRecords = new ArrayList<>(records.size() - count);
            }

            remainingRecords.add(record);
        }
        count++;
    }

    // record the restored offset for its change log partition
    restoredOffsets.put(storePartition, lastOffset + 1);

    return remainingRecords;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public void bufferRecord(ConsumerRecord<Integer, Integer> record) {
    recordBuffer.add(
        new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), 0L,
                             TimestampType.CREATE_TIME, 0L, 0, 0,
                             serializer.serialize(record.topic(), record.key()),
                             serializer.serialize(record.topic(), record.value())));
    endOffset = record.offset();

    super.updateEndOffsets(Collections.singletonMap(assignedPartition, endOffset));
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public static MarkerKey fromRecord(ConsumerRecord r) {
    return new MarkerKey(r.partition(), r.offset());
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@Override
public void execute(Tuple input) {
    ConsumerRecord<String, byte[]> record = (ConsumerRecord<String, byte[]>) input.getValueByField("record");

    long kafkaOffset = record.offset();
    String fromTopic = record.topic();

    FullyOffset currentOffset = new FullyOffset(0, 0, 0);

    try {
        //处理ctrl topic的数据
        if (fromTopic.equalsIgnoreCase(dsInfo.getCtrlTopic())) {
            processControlCommand(record, input);
            return;
        }

        // a 读取message数据
        processor.preProcess(record);

        // b 一次读取一个partition
        List<DispatcherPackage> list;
        int partitionOffset = 0;


        do {
            partitionOffset++;
            list = processor.getNextList();
            if (list == null) {
                break;
            }

            //分schema后的子包
            int subOffset = 1;
            for (DispatcherPackage subPackage : list) {
                currentOffset = new FullyOffset(kafkaOffset, partitionOffset, subOffset);

                // 1 获得数据
                String key = subPackage.getKey();
                byte[] content = subPackage.getContent();
                int msgCount = subPackage.getMsgCount();
                String schemaName = subPackage.getSchemaName();
                String toTopic = subPackage.getToTopic();

                ContinuousFullyOffset continuousOffset = getSchemaFullyOffset(schemaName);
                continuousOffset.setProcessingOffset(currentOffset);
                if (key == null) {
                    // 2 构建数据消息的key，记录上一个offset是谁, 主要是用于日志查错
                    subPackage.setKey(continuousOffset.toString());
                    key = subPackage.getKey();
                }

                logger.debug(String.format("  currentOffset=%s, from_topic: %s, (to_topic:%s, schemaName=%s), Key=%s, msg_count=%d",
                        currentOffset.toString(), fromTopic, toTopic, schemaName, key, msgCount));
                this.collector.emit(input, new Values(subPackage, currentOffset));

                continuousOffset.setProcessedOffset(currentOffset);

                subOffset++;
            }
        } while (true);

        this.collector.ack(input);

    } catch (Exception ex) {
        // Print something in the log
        logger.error(String.format("FAIL! Dispatcher bolt fails at offset (%s).", currentOffset.toString()));
        // Call fail
        this.collector.fail(input);

        collector.reportError(ex);
        throw new RuntimeException(ex);
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public QueueElement(ConsumerRecord<String, byte[]> record) {
    this.key = record.offset();
    setRecord(record);
}