Java ConsumerRecord.partition方法代码示例

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
static Message<? extends com.google.protobuf.Message> fromKafka(com.google.protobuf.Message protoMessage, Envelope envelope, ConsumerRecord<String, byte[]> record) {
    boolean wasReceived = true;

    Topic topic = new Topic(record.topic());
    String partitioningKey = record.key();
    int partitionId = record.partition();
    long offset = record.offset();

    String messageId = envelope.getMessageId();
    String correlationId = envelope.getCorrelationId();

    MessageType type = MessageType.of(protoMessage);

    String requestCorrelationId = envelope.getRequestCorrelationId();
    Topic replyTo = new Topic(envelope.getReplyTo());

    Metadata meta = new Metadata(wasReceived, topic, partitioningKey, partitionId, offset, messageId, correlationId, requestCorrelationId, replyTo, type);
    return new Message<>(protoMessage, meta);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private synchronized void onMessageConsumed(ConsumerRecord<String, String> record) {
    log.info(String.format("Consumed message: [%s]", record));

    TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());

    if (!messagesByTopicPartition.containsKey(topicPartition)) {
        messagesByTopicPartition.put(topicPartition, new VersionedMessages(Lists.newLinkedList()));
    }

    VersionedMessages versionedMessages = messagesByTopicPartition.get(topicPartition);
    LinkedList<ConsumerRecord<String, String>> messages = versionedMessages.messages;
    messages.addFirst(record);

    if (messages.size() > maxTopicMessagesCount) {
        messages.removeLast();
    }

    versionedMessages.version.incrementAndGet();
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@Override
public Boolean call() {

    logger.debug("Number of records received : {}", records.count());
    try {
        for (final ConsumerRecord<String, Serializable> record : records) {
            TopicPartition tp = new TopicPartition(record.topic(), record.partition());
            logger.info("Record received topicPartition : {}, offset : {}", tp,
                record.offset());
            partitionToUncommittedOffsetMap.put(tp, record.offset());

            processConsumerRecords(record);
        }
    } catch (Exception e) {
        logger.error("Error while consuming", e);
    }
    return true;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private void pollAndCommitTransactionsBatch() {
    ConsumerRecords<ByteBuffer, ByteBuffer> records = consumer.poll(POLL_TIMEOUT);
    List<TransactionScope> scopes = new ArrayList<>(records.count());
    for (ConsumerRecord<ByteBuffer, ByteBuffer> record : records) {
        TransactionScope transactionScope = serializer.deserialize(record.key());
        if (transactionScope.getScope().isEmpty()) {
            LOGGER.warn("[R] {} polled empty transaction {}", readerId, transactionScope.getTransactionId());
        }
        TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition());
        buffer.put(transactionScope.getTransactionId(),
                new TransactionData(transactionScope, record.value(), topicPartition, record.offset()));
        scopes.add(transactionScope);
        committedOffsetMap.computeIfAbsent(topicPartition, COMMITTED_OFFSET).notifyRead(record.offset());
    }
    if (!scopes.isEmpty()) {
        scopes.sort(SCOPE_COMPARATOR);
        LOGGER.trace("[R] {} polled {}", readerId, scopes);
    }
    approveAndCommitTransactionsBatch(scopes);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private void convertMessages(ConsumerRecords<byte[], byte[]> msgs) {
    for (ConsumerRecord<byte[], byte[]> msg : msgs) {
        log.trace("Consuming message with key {}, value {}", msg.key(), msg.value());
        SchemaAndValue keyAndSchema = keyConverter.toConnectData(msg.topic(), msg.key());
        SchemaAndValue valueAndSchema = valueConverter.toConnectData(msg.topic(), msg.value());
        SinkRecord record = new SinkRecord(msg.topic(), msg.partition(),
                keyAndSchema.schema(), keyAndSchema.value(),
                valueAndSchema.schema(), valueAndSchema.value(),
                msg.offset(),
                ConnectUtils.checkAndConvertTimestamp(msg.timestamp()),
                msg.timestampType());
        record = transformationChain.apply(record);
        if (record != null) {
            messageBatch.add(record);
        }
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private void reconsumeLater(ConsumerRecord<String, byte[]> consumeRecord) throws InterruptedException, ExecutionException {

		// add all header to headList except RETRY_COUNT
		Headers headers = consumeRecord.headers();
		List<Header> headerList = new ArrayList<Header>(8);
		Iterator<Header> iterator = headers.iterator();
		Integer retryCount = -1;
		boolean hasOrignalHeader = false;
		while (iterator.hasNext()) {
			Header next = iterator.next();
			if (next.key().equals(RETRY_COUNT_KEY)) {
				retryCount = serializer.deserialize(next.value());
				continue;
			}
			
			if(next.key().equals(ORGINAL_TOPIC)){
				hasOrignalHeader = true;
			}
			headerList.add(next);
		}
		
		// add RETRY_COUNT to header
		retryCount++;
		headerList.add(new RecordHeader(RETRY_COUNT_KEY, serializer.serialization(retryCount)));
		
		if(!hasOrignalHeader){
			headerList.add(new RecordHeader(ORGINAL_TOPIC, serializer.serialization(consumeRecord.topic())));
		}

		// send message to corresponding queue according to retry times
		String retryTopic = calcRetryTopic(consumeRecord.topic(), retryCount);
		
		ProducerRecord<String, byte[]> record = new ProducerRecord<>(retryTopic,
				consumeRecord.partition() % retryQueuePartitionCount.get(retryTopic), null, consumeRecord.key(),
				consumeRecord.value(), headerList);
		Future<RecordMetadata> publishKafkaMessage = retryQueueMsgProducer.publishKafkaMessage(record);
		publishKafkaMessage.get();
	}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public static <K, V> Payload<K, V> decodePayload(Deserializer<V> valueDeserializer, ConsumerRecord<K, byte[]> originConsumerRecord) {
    TracingHeader tracingHeader = null;
    ConsumerRecord<K, V> dataRecord = null;
    boolean sampled = false;
    byte[] data = originConsumerRecord.value();
    byte[] vData = null;
    if (data.length <= HEADER_LENGTH) {
        vData = data;
    } else {
        ByteBuffer byteBuf = ByteBuffer.wrap(data);
        short magic = byteBuf.getShort(0);
        short tpLen = byteBuf.getShort(2);
        if (magic == MAGIC && tpLen == TracingHeader.LENGTH) {
            byte[] tpBytes = new byte[tpLen];
            System.arraycopy(byteBuf.array(), HEADER_LENGTH, tpBytes, 0, tpLen);
            tracingHeader = TracingHeader.fromBytes(tpBytes);
            sampled = true;
            int dataOffset = tpLen + HEADER_LENGTH;
            vData = new byte[byteBuf.array().length - dataOffset];
            System.arraycopy(byteBuf.array(), dataOffset, vData, 0, vData.length);
        } else {
            vData = data;
        }
    }
    dataRecord = new ConsumerRecord<>(originConsumerRecord.topic(),
            originConsumerRecord.partition(), originConsumerRecord.offset(),
            originConsumerRecord.key(), valueDeserializer.deserialize(originConsumerRecord.topic(), vData));
    return new Payload<>(tracingHeader, dataRecord, sampled);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private List<KafkaResult> consume() {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords consumerRecords = kafkaConsumer.poll(clientConfig.getPollTimeoutMs());

    logger.info("Consumed {} records", consumerRecords.count());
    final Iterator<ConsumerRecord> recordIterator = consumerRecords.iterator();
    while (recordIterator.hasNext()) {
        // Get next record
        final ConsumerRecord consumerRecord = recordIterator.next();

        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit();
    return kafkaResultList;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public TransactionWrapper(ConsumerRecord<ByteBuffer, ByteBuffer> record, TransactionMetadata deserializedMetadata) {
    GridArgumentCheck.notNull(deserializedMetadata, "metadata cannot be null");
    this.value = record.value();
    this.key = record.key();
    this.topicPartition = new TopicPartition(record.topic(), record.partition());
    this.offset = record.offset();
    this.deserializedMetadata = deserializedMetadata;
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@Override
public void onCompletion(Throwable error, ConsumerRecord<String, String> record) {
    TopicPartition partition = new TopicPartition(record.topic(), record.partition());
    List<ConsumerRecord<String, String>> records = consumedRecords.get(partition);
    if (records == null) {
        records = new ArrayList<>();
        consumedRecords.put(partition, records);
    }
    records.add(record);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void update(final ConsumerRecord<byte[], byte[]> record) {
    final SourceNodeAndDeserializer sourceNodeAndDeserializer = deserializers.get(record.topic());
    final ConsumerRecord<Object, Object> deserialized = sourceNodeAndDeserializer.deserializer.deserialize(record);
    final ProcessorRecordContext recordContext =
            new ProcessorRecordContext(deserialized.timestamp(),
                                       deserialized.offset(),
                                       deserialized.partition(),
                                       deserialized.topic());
    processorContext.setRecordContext(recordContext);
    processorContext.setCurrentNode(sourceNodeAndDeserializer.sourceNode);
    sourceNodeAndDeserializer.sourceNode.process(deserialized.key(), deserialized.value());
    offsets.put(new TopicPartition(record.topic(), record.partition()), deserialized.offset() + 1);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
@Override
public void update(final ConsumerRecord<byte[], byte[]> record) {
    final TopicPartition tp = new TopicPartition(record.topic(), record.partition());
    if (!updatedPartitions.containsKey(tp)) {
        updatedPartitions.put(tp, 0);
    }
    updatedPartitions.put(tp, updatedPartitions.get(tp) + 1);
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
private static void addRecord(final ConsumerRecord<byte[], byte[]> record,
                              final Map<String, Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>>> recordPerTopicPerPartition) {

    final String topic = record.topic();
    final TopicPartition partition = new TopicPartition(topic, record.partition());

    if ("data".equals(topic)
        || "echo".equals(topic)
        || "min".equals(topic)
        || "sum".equals(topic)) {

        Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> topicRecordsPerPartition
            = recordPerTopicPerPartition.get(topic);

        if (topicRecordsPerPartition == null) {
            topicRecordsPerPartition = new HashMap<>();
            recordPerTopicPerPartition.put(topic, topicRecordsPerPartition);
        }

        List<ConsumerRecord<byte[], byte[]>> records = topicRecordsPerPartition.get(partition);
        if (records == null) {
            records = new ArrayList<>();
            topicRecordsPerPartition.put(partition, records);
        }
        records.add(record);
    } else {
        throw new RuntimeException("FAIL: received data from unexpected topic: " + record);
    }
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
public static MarkerKey fromRecord(ConsumerRecord r) {
    return new MarkerKey(r.partition(), r.offset());
}
 

import org.apache.kafka.clients.consumer.ConsumerRecord; //导入方法依赖的package包/类
/**
 * 处理mysql 拉全量请求
 */
@Override
public void process(final ConsumerRecord<String, byte[]> consumerRecord, Object... args){
	try{
 	List<MessageEntry> msgEntryLst = parser.getEntry(consumerRecord.value());
 	if(msgEntryLst.isEmpty())
 		return ; 
 	EntryHeader entryHeader = msgEntryLst.get(0).getEntryHeader();
 	EventType operType = entryHeader.getOperType();
 	
 	//TODO 暂时放弃拉全量表的update/delete等消息
     if (operType!=EventType.INSERT) {
         listener.reduceFlowSize(consumerRecord.serializedValueSize());
         consumerListener.syncOffset(consumerRecord);
         return;
     }
     
  // 判断是否处理过该消息
     String msgPos = entryHeader.getPos();
     Object processed = cache.getIfPresent(msgPos);
     if (processed != null) {
         logger.info("Data have bean processed, the data position is [{}]", msgPos);
         listener.reduceFlowSize(consumerRecord.serializedValueSize());
         consumerListener.syncOffset(consumerRecord);
         return;
     }
     
     logger.info("Received FULL DATA PULL REQUEST message");
     ControlMessage message = Convertor.mysqlFullPullMessage(msgEntryLst.get(0), listener.getListenerId(), consumerRecord);
     String schemaName = getStringValue("SCHEMA_NAME", message);
     String tableName = getStringValue("TABLE_NAME", message);
     DataTable table = ThreadLocalCache.get(Constants.CacheNames.DATA_TABLES, Utils.buildDataTableCacheKey(schemaName, tableName));
     if (table == null) {
         logger.warn("Table {}.{} is not supported,please configure it in dbus database.", schemaName, tableName);
         return;
     }
     
     for (String controlTopic : controlTopics) {
         String json = message.toJSONString();
         ProducerRecord<String, byte[]> producerRecord = new ProducerRecord<>(controlTopic, message.getType(), json.getBytes());
	
         Future<RecordMetadata> future = listener.sendRecord(producerRecord);
         future.get();
	
         logger.info("write initial load request message to kafka: {}", json);
     }
     
  // 暂停接收 topic 的数据
        TopicPartition tp = new TopicPartition(consumerRecord.topic(), consumerRecord.partition());
        consumerListener.pauseTopic(tp, consumerRecord.offset(), message);

        // emit FULL_DATA_PULL_REQ 通知给bolt
        EmitData emitData = new EmitData();
        emitData.add(EmitData.DB_SCHEMA, schemaName);
        emitData.add(EmitData.DATA_TABLE, tableName);

        List<Object> values = new Values(emitData, Command.FULL_DATA_PULL_REQ);
        listener.emitData(values, consumerRecord);

        consumerListener.syncOffset(consumerRecord);
        cache.put(msgPos, msgEntryLst.get(0).toString());
	}catch(Exception e){
		throw new RuntimeException(e);
	}
}