Java AmazonKinesis类代码示例

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
private static List<Shard> getShards(AmazonKinesis client, String stream) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(stream);
    List<Shard> shards = new ArrayList<>();
    String exclusiveStartShardId = null;
    do {
        describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
        DescribeStreamResult describeStreamResult = client.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
            exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
        } else {
            exclusiveStartShardId = null;
        }
    } while (exclusiveStartShardId != null);

    return shards;
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());
	String name = "test-stream";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ConsumerDestination destination =
			provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
@BeforeEach
public void before() {
  this.sourceTaskContext = mock(SourceTaskContext.class);
  this.offsetStorageReader = mock(OffsetStorageReader.class);
  when(this.sourceTaskContext.offsetStorageReader()).thenReturn(this.offsetStorageReader);
  this.task = new KinesisSourceTask();
  this.task.initialize(this.sourceTaskContext);
  this.kinesisClient = mock(AmazonKinesis.class);
  this.task.time = mock(Time.class);
  this.task.kinesisClientFactory = mock(KinesisClientFactory.class);
  when(this.task.kinesisClientFactory.create(any())).thenReturn(this.kinesisClient);

  this.settings = TestData.settings();
  this.config = new KinesisSourceConnectorConfig(this.settings);

}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
KinesisStreamFactory(Builder builder) {
  this.stream = builder.stream;
  this.app = builder.app;
  this.regionName = builder.awsRegion;
  this.endpoint = builder.awsEndpoint != null ?
      builder.awsEndpoint :
      Region.getRegion(Regions.fromName(regionName))
          .getServiceEndpoint(AmazonKinesis.ENDPOINT_PREFIX);

  this.checkpointInterval = builder.checkpointInterval;
  this.initialPositionInStream = builder.initialPositionInStream;
  this.storageLevel = builder.storageLevel;

  this.awsAccessKeyId = builder.awsAccessKeyId;
  this.awsSecretKey = builder.awsSecretKey;
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
@Override
public AmazonKinesis getKinesisClient() {
  return new AmazonKinesisMock(transform(shardedData,
      new Function<List<TestData>, List<Record>>() {

        @Override
        public List<Record> apply(@Nullable List<TestData> testDatas) {
          return transform(testDatas, new Function<TestData, Record>() {

            @Override
            public Record apply(@Nullable TestData testData) {
              return testData.convertToRecord();
            }
          });
        }
      }), numberOfRecordsPerGet);

}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
/**
 * Creates an AmazonKinesis client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @return a new AmazonKinesis client
 */
public static AmazonKinesis createKinesisClient(Properties configProps) {
	// set a Flink-specific user agent
	ClientConfiguration awsClientConfig = new ClientConfigurationFactory().getConfig()
			.withUserAgentPrefix(String.format(USER_AGENT_FORMAT,
													EnvironmentInformation.getVersion(),
													EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig)
			.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	}
	return builder.build();
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    verify(args);
    String stream = args[0];
    Region region = RegionUtils.getRegion(args[1]);

    AWSCredentials credentials = getCreds();
    AmazonKinesis client = new AmazonKinesisClient(credentials, CLIENT_CONF);
    client.setRegion(region);
    checkStream(client.describeStream(stream));

    System.out.println("Let's start putting records!");
    Random rnd = new Random(System.currentTimeMillis());
    for (;;) {
        putEventRecord(client, stream);
        Thread.sleep(rnd.nextInt(500) + 650);
    }
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
/**
 * Collect data for Kinesis.
 *
 * @param stats
 *            current statistics object.
 * @param account
 *            currently used credentials object.
 * @param region
 *            currently used aws region.
 */
public static void scanKinesis(AwsStats stats, AwsAccount account, Regions region) {
	LOG.debug("Scan for Kinesis in region " + region.getName() + " in account " + account.getAccountId());

	try {
		AmazonKinesis kinesis = new AmazonKinesisClient(account.getCredentials());
		kinesis.setRegion(Region.getRegion(region));

		List<String> list = kinesis.listStreams().getStreamNames();

		int totalItems = list.size();
		for (String streamName : list) {
			stats.add(new AwsResource(streamName, account.getAccountId(), AwsResourceType.Kinesis, region));
		}

		LOG.info(totalItems + " Kinesis streams in region " + region.getName() + " in account " + account.getAccountId());
	} catch (AmazonServiceException ase) {
		LOG.error("Exception of Kinesis: " + ase.getMessage());
	}
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
public static List<com.amazonaws.services.kinesis.model.Record> getPreviewRecords(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    int maxBatchSize,
    GetShardIteratorRequest getShardIteratorRequest
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
  String shardIterator = getShardIteratorResult.getShardIterator();

  GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
  getRecordsRequest.setShardIterator(shardIterator);
  getRecordsRequest.setLimit(maxBatchSize);

  GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
  return getRecordsResult.getRecords();
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
private void run(final int events, final OutputFormat format,
		final String streamName, final String region) throws Exception {
	AmazonKinesis kinesisClient = new AmazonKinesisClient(
			new DefaultAWSCredentialsProviderChain());
	kinesisClient.setRegion(Region.getRegion(Regions.fromName(region)));
	int count = 0;
	SensorReading r = null;
	do {
		r = nextSensorReading(format);

		try {
			PutRecordRequest req = new PutRecordRequest()
					.withPartitionKey("" + rand.nextLong())
					.withStreamName(streamName)
					.withData(ByteBuffer.wrap(r.toString().getBytes()));
			kinesisClient.putRecord(req);
		} catch (ProvisionedThroughputExceededException e) {
			Thread.sleep(BACKOFF);
		}

		System.out.println(r);
		count++;
	} while (count < events);
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
private static String getShardIterator(AmazonKinesis client, String stream, Shard shard, String start) {
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());

    if (!Strings.isNullOrEmpty(start)) {
        getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_TIMESTAMP);
        getShardIteratorRequest.setTimestamp(new Date(System.currentTimeMillis() - Duration.parse(start).toMillis()));
    } else {
        getShardIteratorRequest.setShardIteratorType(ShardIteratorType.LATEST);
    }

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
private static String getShardIteratorAtSequenceNumber(AmazonKinesis client, String stream, Shard shard, String sequenceNumber) {
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());

    getShardIteratorRequest.setShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER);
    getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber);

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
private static String getOldestShardIterator(AmazonKinesis client, String stream, Shard shard) {
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(stream);
    getShardIteratorRequest.setShardId(shard.getShardId());
    getShardIteratorRequest.setShardIteratorType(ShardIteratorType.TRIM_HORIZON);

    GetShardIteratorResult getShardIteratorResult = client.getShardIterator(getShardIteratorRequest);
    return getShardIteratorResult.getShardIterator();
}
 

import com.amazonaws.services.kinesis.AmazonKinesis; //导入依赖的package包/类
@Override
protected void createAWSClient()
{
    client = tryClientFactory(config.clientFactoryMethod, AmazonKinesis.class, true);
    if ((client == null) && (config.clientEndpoint == null))
    {
        client = tryClientFactory("com.amazonaws.services.kinesis.AmazonKinesisClientBuilder.defaultClient", AmazonKinesis.class, false);
    }
    if (client == null)
    {
        LogLog.debug(getClass().getSimpleName() + ": creating service client via constructor");
        client = tryConfigureEndpointOrRegion(new AmazonKinesisClient(), config.clientEndpoint);
    }
}