Java DescribeStreamRequest类代码示例

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
private void refreshShards() {
  DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
  describeStreamRequest.setStreamName(streamName);
  String exclusiveStartShardId = null;
  List<Shard> shards = new ArrayList<>();

  do {
    describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
    DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
    shards.addAll(describeStreamResult.getStreamDescription().getShards());

    if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
      exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
    } else {
      exclusiveStartShardId = null;
    }
  } while (exclusiveStartShardId != null);

  this.shards = shards;
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());
	String name = "test-stream";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ConsumerDestination destination =
			provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Before
public void setup() throws Exception {
    KinesisEndpoint endpoint = new KinesisEndpoint(null, "streamName", component);
    endpoint.setAmazonKinesisClient(kinesisClient);
    endpoint.setIteratorType(ShardIteratorType.LATEST);
    undertest = new KinesisConsumer(endpoint, processor);

    when(kinesisClient.getRecords(any(GetRecordsRequest.class)))
        .thenReturn(new GetRecordsResult()
            .withNextShardIterator("nextShardIterator")
        );
    when(kinesisClient.describeStream(any(DescribeStreamRequest.class)))
        .thenReturn(new DescribeStreamResult()
            .withStreamDescription(new StreamDescription()
                .withShards(new Shard().withShardId("shardId"))
            )
        );
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class)))
        .thenReturn(new GetShardIteratorResult()
            .withShardIterator("shardIterator")
        );
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void itObtainsAShardIteratorOnFirstPollForSequenceNumber() throws Exception {
    undertest.getEndpoint().setSequenceNumber("12345");
    undertest.getEndpoint().setIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER);

    undertest.poll();

    final ArgumentCaptor<DescribeStreamRequest> describeStreamReqCap = ArgumentCaptor.forClass(DescribeStreamRequest.class);
    final ArgumentCaptor<GetShardIteratorRequest> getShardIteratorReqCap = ArgumentCaptor.forClass(GetShardIteratorRequest.class);

    verify(kinesisClient).describeStream(describeStreamReqCap.capture());
    assertThat(describeStreamReqCap.getValue().getStreamName(), is("streamName"));

    verify(kinesisClient).getShardIterator(getShardIteratorReqCap.capture());
    assertThat(getShardIteratorReqCap.getValue().getStreamName(), is("streamName"));
    assertThat(getShardIteratorReqCap.getValue().getShardId(), is("shardId"));
    assertThat(getShardIteratorReqCap.getValue().getShardIteratorType(), is("AFTER_SEQUENCE_NUMBER"));
    assertThat(getShardIteratorReqCap.getValue().getStartingSequenceNumber(), is("12345"));

}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
/**
 * Create the specified topic with the specified number of partitions
 */
public void createTopic(String topicName, int partitions) {
	LOGGER.info("Determining if  Kinesis topic: {} already exists...", topicName);
	try{
		final DescribeStreamRequest describeRequest = new DescribeStreamRequest();
		describeRequest.withStreamName(topicName);
		this.client.describeStream(describeRequest);
		
	}catch(ResourceNotFoundException rnf){
		LOGGER.info("Kinesis stream for topic: {} does not exist, creating now with shard count: {}",topicName, partitions);
		final CreateStreamRequest request = new CreateStreamRequest();
		request.withStreamName(topicName);
		request.withShardCount(partitions);
		this.client.createStream(request);
	    this.waitForStreamToBecomeAvailable(topicName, DEFAULT_WAIT_TIME_MINUTES);
		LOGGER.info("Create topic completed for topic: {}", topicName);
	}    
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
private static void waitForStreamToBecomeAvailable(String myStreamName) throws InterruptedException {
	LOG.info("Waiting for " + myStreamName + " to become ACTIVE...");

	long startTime = System.currentTimeMillis();
	long endTime = startTime + TimeUnit.MINUTES.toMillis(5);
	while (System.currentTimeMillis() < endTime) {
		Thread.sleep( TimeUnit.SECONDS.toMillis(5));
		DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
		describeStreamRequest.setStreamName(myStreamName);
		describeStreamRequest.setLimit(10);
		DescribeStreamResult describeStreamResponse = kinesisClient
				.describeStream(describeStreamRequest);

		String streamStatus = describeStreamResponse
				.getStreamDescription().getStreamStatus();
		if ("ACTIVE".equals(streamStatus)) {
			return;
		}
	}
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
private List<Shard> describeStream(String stream) {
	AmazonKinesisAsync amazonKinesis = localKinesisResource.getResource();

	String exclusiveStartShardId = null;

	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
			.withStreamName(stream);

	List<Shard> shardList = new ArrayList<>();

	while (true) {
		DescribeStreamResult describeStreamResult = null;

		describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
		describeStreamResult = amazonKinesis.describeStream(describeStreamRequest);
		StreamDescription streamDescription = describeStreamResult.getStreamDescription();
		if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
			shardList.addAll(streamDescription.getShards());

			if (streamDescription.getHasMoreShards()) {
				exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
				continue;
			}
			else {
				return shardList;
			}
		}
		try {
			Thread.sleep(100);
		}
		catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			throw new IllegalStateException(e);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionConsumerExistingStreamUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor =
			ArgumentCaptor.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream =
			describeStreamResultWithShards(Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream)
			.thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1))
			.updateShardCount(updateShardCaptor.capture());

	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType()).isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount()).isEqualTo(targetShardCount);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());

	String name = "test-stream";
	Integer shards = 1;

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, shards))
			.thenReturn(new CreateStreamResult());

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock)
			.createStream(name, shards);

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionProducerUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream =
			describeStreamResultWithShards(Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream)
			.thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1))
			.updateShardCount(updateShardCaptor.capture());
	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType()).isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount()).isEqualTo(targetShardCount);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));


	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
			.thenReturn(new CreateStreamResult());

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock)
			.createStream(name, instanceCount * concurrency);

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Override
public void start(Map<String, String> settings) {
  log.info("start()");
  this.settings = settings;
  this.config = new KinesisSourceConnectorConfig(settings);
  this.kinesisClient = this.kinesisClientFactory.create(this.config);

  DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
      .withStreamName(this.config.kinesisStreamName);

  DescribeStreamResult describeStreamResult = this.kinesisClient.describeStream(describeStreamRequest);
  this.streamDescription = describeStreamResult.getStreamDescription();
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
@Test
public void start() {
  final DescribeStreamRequest expectedDescribeStreamRequest = new DescribeStreamRequest()
      .withStreamName(TestData.EXPECTED_STREAM_NAME);

  final int SHARD_COUNT = 50;
  List<Shard> shards = new ArrayList<>(SHARD_COUNT);
  for (int i = 0; i < SHARD_COUNT; i++) {
    String shardId = String.format("%03d", i);
    final Shard shard = new Shard()
        .withShardId(shardId);
    shards.add(shard);
  }


  final StreamDescription streamDescription = new StreamDescription()
      .withStreamName(TestData.EXPECTED_STREAM_NAME)
      .withShards(shards);

  final DescribeStreamResult expectedStreamRequest = new DescribeStreamResult()
      .withStreamDescription(streamDescription);

  when(this.kinesisClient.describeStream(any(DescribeStreamRequest.class))).thenReturn(expectedStreamRequest);
  this.connector.start(TestData.settings());

  List<Map<String, String>> taskConfigs = this.connector.taskConfigs(SHARD_COUNT);
  assertEquals(SHARD_COUNT, taskConfigs.size());
  verify(this.kinesisClient, atLeastOnce()).describeStream(expectedDescribeStreamRequest);
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入依赖的package包/类
private String getShardItertor() {
    // either return a cached one or get a new one via a GetShardIterator request.
    if (currentShardIterator == null) {
        String shardId;

        //If ShardId supplied use it, else choose first one
        if (!getEndpoint().getShardId().isEmpty()) {
            shardId = getEndpoint().getShardId();
        } else {
            DescribeStreamRequest req1 = new DescribeStreamRequest()
                    .withStreamName(getEndpoint().getStreamName());
            DescribeStreamResult res1 = getClient().describeStream(req1);
            shardId = res1.getStreamDescription().getShards().get(0).getShardId();
        }
        LOG.debug("ShardId is: {}", shardId);

        GetShardIteratorRequest req = new GetShardIteratorRequest()
                .withStreamName(getEndpoint().getStreamName())
                .withShardId(shardId)
                .withShardIteratorType(getEndpoint().getIteratorType());

        if (hasSequenceNumber()) {
            req.withStartingSequenceNumber(getEndpoint().getSequenceNumber());
        }

        GetShardIteratorResult result = getClient().getShardIterator(req);
        currentShardIterator = result.getShardIterator();
    }
    LOG.debug("Shard Iterator is: {}", currentShardIterator);
    return currentShardIterator;
}