Java Record类代码示例

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records,
		IRecordProcessorCheckpointer checkpointer) {
	LOG.info(String.format("Received %s Records", records.size()));
	
	// add a call to your business logic here!
	//
	// myLinkedClasses.doSomething(records)
	//
	//
	try {
		checkpointer.checkpoint();
	} catch (KinesisClientLibDependencyException | InvalidStateException
			| ThrottlingException | ShutdownException e) {
		e.printStackTrace();
		super.shutdown(checkpointer, ShutdownReason.ZOMBIE);
	}
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
private KinesisWrapper(final InternalEvent internal) {
  KinesisEventRecord eventRecord = ((KinesisInternalEvent) internal).getRecord();
  Record record = eventRecord.getKinesis();

  this.partitionKey = record.getPartitionKey();
  this.sequenceNumber = record.getSequenceNumber();
  this.eventSource = eventRecord.getEventSource();
  this.sourceArn = eventRecord.getEventSourceARN();
  this.functionName = internal.getCtx().getFunctionName();
  this.functionVersion = internal.getCtx().getFunctionVersion();
  this.processingTime = System.currentTimeMillis();
  this.arrivalTime = record.getApproximateArrivalTimestamp().getTime();
  this.timestamp = internal.getEventTime();
  this.processingDelay = processingTime - timestamp;

  if (internal.getEventObj() != null) {
    this.payload = internal.getEventObj().getPayload();
  } else {
    this.payload = null;
  }
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@SneakyThrows
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    List<Record> records = processRecordsInput.getRecords();
    // Used to update the last processed record
    IRecordProcessorCheckpointer checkpointer = processRecordsInput.getCheckpointer();
    log.info("Recovering records from kinesis.");
    for (Record r : records) {
        try {
            int len = r.getData().remaining();
            byte[] buffer = new byte[len];
            r.getData().get(buffer);
            String json = new String(buffer, "UTF-8");
            ZombieLecture lecture = mapper.readValue(json, ZombieLecture.class);
            this.processZombieLecture(lecture);
            log.debug(processedRecords++ + ": " + json);
            if (processedRecords % 1000 == 999) {
                // Uncomment next line to keep track of the processed lectures. 
                checkpointer.checkpoint();
            }
        } catch (UnsupportedEncodingException | MessagingException ex) {
            log.warn(ex.getMessage());
        }
    }
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Override
public AmazonKinesis getKinesisClient() {
  return new AmazonKinesisMock(transform(shardedData,
      new Function<List<TestData>, List<Record>>() {

        @Override
        public List<Record> apply(@Nullable List<TestData> testDatas) {
          return transform(testDatas, new Function<TestData, Record>() {

            @Override
            public Record apply(@Nullable TestData testData) {
              return testData.convertToRecord();
            }
          });
        }
      }), numberOfRecordsPerGet);

}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Test
public void recordsAreSentToTheProcessor() throws Exception {
    when(kinesisClient.getRecords(any(GetRecordsRequest.class)))
        .thenReturn(new GetRecordsResult()
            .withNextShardIterator("nextShardIterator")
            .withRecords(new Record().withSequenceNumber("1"), new Record().withSequenceNumber("2"))
        );

    int messageCount = undertest.poll();

    assertThat(messageCount, is(2));
    final ArgumentCaptor<Exchange> exchangeCaptor = ArgumentCaptor.forClass(Exchange.class);

    verify(processor, times(2)).process(exchangeCaptor.capture(), any(AsyncCallback.class));
    assertThat(exchangeCaptor.getAllValues().get(0).getIn().getBody(Record.class).getSequenceNumber(), is("1"));
    assertThat(exchangeCaptor.getAllValues().get(1).getIn().getBody(Record.class).getSequenceNumber(), is("2"));
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Test
public void exchangePropertiesAreSet() throws Exception {
    String partitionKey = "partitionKey";
    String sequenceNumber = "1";
    when(kinesisClient.getRecords(any(GetRecordsRequest.class)))
        .thenReturn(new GetRecordsResult()
            .withNextShardIterator("nextShardIterator")
            .withRecords(new Record()
                .withSequenceNumber(sequenceNumber)
                .withApproximateArrivalTimestamp(new Date(42))
                .withPartitionKey(partitionKey)
            )
        );

    undertest.poll();

    final ArgumentCaptor<Exchange> exchangeCaptor = ArgumentCaptor.forClass(Exchange.class);

    verify(processor).process(exchangeCaptor.capture(), any(AsyncCallback.class));
    assertThat(exchangeCaptor.getValue().getIn().getHeader(KinesisConstants.APPROX_ARRIVAL_TIME, long.class), is(42L));
    assertThat(exchangeCaptor.getValue().getIn().getHeader(KinesisConstants.PARTITION_KEY, String.class), is(partitionKey));
    assertThat(exchangeCaptor.getValue().getIn().getHeader(KinesisConstants.SEQUENCE_NUMBER, String.class), is(sequenceNumber));
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
/**
 * Implement InputOperator Interface.
 */
@Override
public void emitTuples()
{
  if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
    return;
  }
  int count = consumer.getQueueSize();
  if (maxTuplesPerWindow > 0) {
    count = Math.min(count, maxTuplesPerWindow - emitCount);
  }
  for (int i = 0; i < count; i++) {
    Pair<String, Record> data = consumer.pollRecord();
    String shardId = data.getFirst();
    String recordId = data.getSecond().getSequenceNumber();
    emitTuple(data);
    MutablePair<String, Integer> shardOffsetAndCount = currentWindowRecoveryState.get(shardId);
    if (shardOffsetAndCount == null) {
      currentWindowRecoveryState.put(shardId, new MutablePair<String, Integer>(recordId, 1));
    } else {
      shardOffsetAndCount.setRight(shardOffsetAndCount.right + 1);
    }
    shardPosition.put(shardId, recordId);
  }
  emitCount += count;
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
public String processNextIterator(String iterator)
{
  GetRecordsRequest getRequest = new GetRecordsRequest();
  getRequest.setLimit(1000);

  getRequest.setShardIterator(iterator);
  // call "get" operation and get everything in this shard range
  GetRecordsResult getResponse = client.getRecords(getRequest);

  iterator = getResponse.getNextShardIterator();

  List<Record> records = getResponse.getRecords();
  processResponseRecords(records);

  return iterator;
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
protected void processResponseRecords( List<Record> records )
{
  if ( records == null || records.isEmpty() ) {
    return;
  }
  receiveCount += records.size();
  logger.debug("ReceiveCount= {}", receiveCount);

  for ( Record record : records ) {
    holdingBuffer.add(record);
    if ( shouldProcessRecord ) {
      processRecord( record );
    }

    if ( doneLatch != null ) {
      doneLatch.countDown();
    }
  }
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Override
protected void processRecord(Record record)
{
  String partitionKey = record.getPartitionKey();
  ByteBuffer data = record.getData();
  logger.info("partitionKey={} ", partitionKey);
  byte[] dataBytes = new byte[data.remaining()];
  data.get(dataBytes, 0, dataBytes.length);

  long key = Long.valueOf(partitionKey);
  TestPOJO expected = new TestPOJO(key);

  TestPOJO read = (TestPOJO)fieldValueGenerator.deserializeObject(dataBytes);

  if (!read.outputFieldsEquals(expected)) {
    logger.error("read is not same as expected. read={}, expected={}", read, expected);
    Assert.assertTrue(false);
  } else {
    logger.info("read is same as expected. read={}, expected={}", read, expected);
  }
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
/**
 * Process a single record.
 *
 * @param record The record to be processed.
 */
private void processSingleRecord(Record record) {

    if (producer == null) {
        producer = createProducer();
    }

    String kafkaServers = System.getenv().get("KAFKA_SERVERS");
    if (null == kafkaServers) {
        kafkaServers = Constants.KAFKA_SERVERS;
    }

    ByteBuffer data = record.getData();


    retryCounter = 0;
    // sending event to Queue
    ProducerRecord<String, byte[]> keyedMessage = new ProducerRecord<>(
            TestConstants.topic,
            TestConstants.partition, data.array());
    this.sendMessage(keyedMessage);
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
private void verifyRecords(List<IncomingMessageEnvelope> outputRecords, List<Record> inputRecords, String shardId) {
  Iterator outputRecordsIter = outputRecords.iterator();
  inputRecords.forEach(record -> {
      IncomingMessageEnvelope envelope = (IncomingMessageEnvelope) outputRecordsIter.next();
      String outputKey = (String) envelope.getKey();
      KinesisIncomingMessageEnvelope kinesisMessageEnvelope = (KinesisIncomingMessageEnvelope) envelope;
      Assert.assertEquals(outputKey, record.getPartitionKey());
      Assert.assertEquals(kinesisMessageEnvelope.getSequenceNumber(), record.getSequenceNumber());
      Assert.assertEquals(kinesisMessageEnvelope.getApproximateArrivalTimestamp(),
          record.getApproximateArrivalTimestamp());
      Assert.assertEquals(kinesisMessageEnvelope.getShardId(), shardId);
      ByteBuffer outputData = ByteBuffer.wrap((byte[]) kinesisMessageEnvelope.getMessage());
      record.getData().rewind();
      Assert.assertTrue(outputData.equals(record.getData()));
      verifyOffset(envelope.getOffset(), record, shardId);
    });
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
static Map<KinesisRecordProcessor, List<Record>> generateRecords(int numRecordsPerShard,
    List<KinesisRecordProcessor> processors) throws ShutdownException, InvalidStateException {
  Map<KinesisRecordProcessor, List<Record>> processorRecordMap = new HashMap<>();
  processors.forEach(processor -> {
      try {
        // Create records and call process records
        IRecordProcessorCheckpointer checkpointer = Mockito.mock(IRecordProcessorCheckpointer.class);
        doNothing().when(checkpointer).checkpoint(anyString());
        doNothing().when(checkpointer).checkpoint();
        ProcessRecordsInput processRecordsInput = Mockito.mock(ProcessRecordsInput.class);
        when(processRecordsInput.getCheckpointer()).thenReturn(checkpointer);
        when(processRecordsInput.getMillisBehindLatest()).thenReturn(1000L);
        List<Record> inputRecords = createRecords(numRecordsPerShard);
        processorRecordMap.put(processor, inputRecords);
        when(processRecordsInput.getRecords()).thenReturn(inputRecords);
        processor.processRecords(processRecordsInput);
      } catch (ShutdownException | InvalidStateException ex) {
        throw new RuntimeException(ex);
      }
    });
  return processorRecordMap;
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
private static List<Record> createRecords(int numRecords) {
  List<Record> records = new ArrayList<>(numRecords);
  Random rand = new Random();

  for (int i = 0; i < numRecords; i++) {
    String dataStr = "testData-" + System.currentTimeMillis();
    ByteBuffer data = ByteBuffer.wrap(dataStr.getBytes(StandardCharsets.UTF_8));
    String key = String.format("partitionKey-%d", rand.nextLong());
    String seqNum = String.format("%04d", 5 * i + 1);
    Record record = new Record()
        .withData(data)
        .withPartitionKey(key)
        .withSequenceNumber(seqNum)
        .withApproximateArrivalTimestamp(new Date());
    records.add(record);
  }
  return records;
}
 

import com.amazonaws.services.kinesis.model.Record; //导入依赖的package包/类
@Test
public void theTransformerShouldFailGracefullyWhenUnableToCompress () {
  CloudWatchMessageModelSumologicTransformer transfomer = new CloudWatchMessageModelSumologicTransformer();
  
  String randomData = "Some random string without GZIP compression";
  ByteBuffer bufferedData = null;
  try {
    bufferedData = encoder.encode(CharBuffer.wrap(randomData));
  } catch (Exception e) {
    Assert.fail("Getting error: "+e.getMessage());
  }
  
  Record mockedRecord = new Record();
  mockedRecord.setData(bufferedData);
  
  CloudWatchLogsMessageModel messageModel = transfomer.toClass(mockedRecord);
  
  
  Assert.assertNull(messageModel);
}