Java IRecordProcessorCheckpointer类代码示例

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records,
		IRecordProcessorCheckpointer checkpointer) {
	LOG.info(String.format("Received %s Records", records.size()));
	
	// add a call to your business logic here!
	//
	// myLinkedClasses.doSomething(records)
	//
	//
	try {
		checkpointer.checkpoint();
	} catch (KinesisClientLibDependencyException | InvalidStateException
			| ThrottlingException | ShutdownException e) {
		e.printStackTrace();
		super.shutdown(checkpointer, ShutdownReason.ZOMBIE);
	}
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer,
		ShutdownReason reason) {
	LOG.info("Shutting down record processor for shard: " + kinesisShardId);

	// Important to checkpoint after reaching end of shard, so we can start
	// processing data from child shards.
	if (reason == ShutdownReason.TERMINATE) {
		try {
			checkpoint(checkpointer);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@SneakyThrows
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    List<Record> records = processRecordsInput.getRecords();
    // Used to update the last processed record
    IRecordProcessorCheckpointer checkpointer = processRecordsInput.getCheckpointer();
    log.info("Recovering records from kinesis.");
    for (Record r : records) {
        try {
            int len = r.getData().remaining();
            byte[] buffer = new byte[len];
            r.getData().get(buffer);
            String json = new String(buffer, "UTF-8");
            ZombieLecture lecture = mapper.readValue(json, ZombieLecture.class);
            this.processZombieLecture(lecture);
            log.debug(processedRecords++ + ": " + json);
            if (processedRecords % 1000 == 999) {
                // Uncomment next line to keep track of the processed lectures. 
                checkpointer.checkpoint();
            }
        } catch (UnsupportedEncodingException | MessagingException ex) {
            log.warn(ex.getMessage());
        }
    }
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
static Map<KinesisRecordProcessor, List<Record>> generateRecords(int numRecordsPerShard,
    List<KinesisRecordProcessor> processors) throws ShutdownException, InvalidStateException {
  Map<KinesisRecordProcessor, List<Record>> processorRecordMap = new HashMap<>();
  processors.forEach(processor -> {
      try {
        // Create records and call process records
        IRecordProcessorCheckpointer checkpointer = Mockito.mock(IRecordProcessorCheckpointer.class);
        doNothing().when(checkpointer).checkpoint(anyString());
        doNothing().when(checkpointer).checkpoint();
        ProcessRecordsInput processRecordsInput = Mockito.mock(ProcessRecordsInput.class);
        when(processRecordsInput.getCheckpointer()).thenReturn(checkpointer);
        when(processRecordsInput.getMillisBehindLatest()).thenReturn(1000L);
        List<Record> inputRecords = createRecords(numRecordsPerShard);
        processorRecordMap.put(processor, inputRecords);
        when(processRecordsInput.getRecords()).thenReturn(inputRecords);
        processor.processRecords(processRecordsInput);
      } catch (ShutdownException | InvalidStateException ex) {
        throw new RuntimeException(ex);
      }
    });
  return processorRecordMap;
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
    LOG.info("Shutting down record processor with shardId: " + shardId + " with reason " + reason);
    if (isShutdown) {
        LOG.warn("Record processor for shardId: " + shardId + " has been shutdown multiple times.");
        return;
    }
    switch (reason) {
        case TERMINATE:
            emit(checkpointer, transformToOutput(buffer.getRecords()));
            try {
                checkpointer.checkpoint();
            } catch (KinesisClientLibDependencyException | InvalidStateException | ThrottlingException | ShutdownException e) {
                LOG.error(e);
            }
            break;
        case ZOMBIE:
            break;
        default:
            throw new IllegalStateException("invalid shutdown reason");
    }
    emitter.shutdown();
    isShutdown = true;
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> arg0, IRecordProcessorCheckpointer arg1) {
	counter += arg0.size();
	if (counter > target) {
		System.out.println("Received : " + counter + " records");
		target += target;
	}
	Record rec;
	for(int i = 0; i < arg0.size(); i++) {
		rec = arg0.get(i);
		try {
			verifyRecord(rec.getData());
		} 
		catch (JSONException | UnsupportedEncodingException e) {
			e.printStackTrace();
		}
	}
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void processRecords(List<Record> records,
		IRecordProcessorCheckpointer checkpointer) {
	LOG.info("Aggregating " + records.size()
			+ " records for Kinesis Shard " + kinesisShardId);
	try {
		// run data into the aggregator
		agg.aggregate(records);

		// checkpoint the aggregator and kcl
		agg.checkpoint();
		checkpointer.checkpoint(records.get(records.size() - 1));

		LOG.debug("Kinesis Checkpoint for Shard " + kinesisShardId
				+ " Complete");
	} catch (Exception e) {
		e.printStackTrace();
		LOG.error(e);
		shutdown(checkpointer, ShutdownReason.ZOMBIE);
	}
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> list, IRecordProcessorCheckpointer irpc) {        
    _logger.info("Processing {} records", list.size());
    for(Record r: list){
        String data = new String(r.getData().array());
        long seq = _buffer.next();
        KinesisEvent evt = _buffer.get(seq);
        evt.setData(data);
        _buffer.publish(seq);
    }
    try{
        irpc.checkpoint();
    }
    catch(InvalidStateException | KinesisClientLibDependencyException | ShutdownException | ThrottlingException ex){
        _logger.warn("Exception while checkpointing", ex);
    }
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    records.forEach(record -> {
        byte[] bytes = new byte[record.getData().remaining()];
        record.getData().get(bytes);
        System.out.write(bytes, 0, bytes.length);
        System.out.println();
    });
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) {
    records.forEach(record -> {
        try {
            byte[] bytes = new byte[record.getData().remaining()];
            record.getData().get(bytes);
            System.out.println(mapper.writeValueAsString(mapper.readTree(bytes)));
        } catch (IOException e) {
            logger.error("", e);
        }
    });
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
@SneakyThrows
public void shutdown(ShutdownInput shutdownInput) {
    IRecordProcessorCheckpointer checkpointer = shutdownInput.getCheckpointer();
    ShutdownReason reason = shutdownInput.getShutdownReason();
    log.info("Finalizado trabajo: {}.", reason);
    checkpointer.checkpoint();
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records, IRecordProcessorCheckpointer iRecordProcessorCheckpointer) {
    try {
        logger.info("Start processing records");

        processRecordsWithRetries(records);

        if (System.currentTimeMillis() > nextCheckpointTimeInMillis) {
            checkpoint(iRecordProcessorCheckpointer);
            nextCheckpointTimeInMillis = System.currentTimeMillis() + CHECKPOINT_INTERVAL_MILLIS;
        }
    } catch (InvalidProtocolBufferException exc) {
        logger.error(exc);
    }
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void shutdown(IRecordProcessorCheckpointer iRecordProcessorCheckpointer, ShutdownReason shutdownReason) {

    if (producer != null) {
        producer.close();
    }

    logger.info("Shutting down record processor for shard: " + shardId);
    // Important to checkpoint after reaching end of shard, so we can start processing data from child shards.
    if (shutdownReason == ShutdownReason.TERMINATE) {
        checkpoint(iRecordProcessorCheckpointer);
    }
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
@Override
public void processRecords(List<Record> records,
                           IRecordProcessorCheckpointer checkpointer) {
    if (!receiver.isStopped()) {
        // this processor works in tandem with the Spark Streaming receiver
        handleRecords(records);
        checkpointIfNeeded(checkpointer);
    }
}
 

import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
    // Important to checkpoint after reaching end of shard,
    // so we can start processing data from child shards.
    if (reason == ShutdownReason.TERMINATE) {
        performCheckpoint(checkpointer);
    }
}