Java Shard类代码示例

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private void refreshShards() {
  DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
  describeStreamRequest.setStreamName(streamName);
  String exclusiveStartShardId = null;
  List<Shard> shards = new ArrayList<>();

  do {
    describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
    DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
    shards.addAll(describeStreamResult.getStreamDescription().getShards());

    if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
      exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
    } else {
      exclusiveStartShardId = null;
    }
  } while (exclusiveStartShardId != null);

  this.shards = shards;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
@Ignore("Kinesalite doesn't support updateShardCount. Test only against real AWS Kinesis")
public void testPartitionCountIncreasedIfAutoAddPartitionsSet() {
	KinesisBinderConfigurationProperties configurationProperties = new KinesisBinderConfigurationProperties();

	String stream = "existing" + System.currentTimeMillis();

	AmazonKinesisAsync amazonKinesis = localKinesisResource.getResource();
	amazonKinesis.createStream(stream, 1);

	List<Shard> shards = describeStream(stream);

	assertThat(shards.size()).isEqualTo(1);

	configurationProperties.setMinShardCount(6);
	configurationProperties.setAutoAddShards(true);
	KinesisTestBinder binder = getBinder(configurationProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	Binding<?> binding = binder.bindConsumer(stream, "test", new NullChannel(), consumerProperties);
	binding.unbind();

	shards = describeStream(stream);

	assertThat(shards.size()).isEqualTo(6);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());
	String name = "test-stream";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ConsumerDestination destination =
			provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public List<Shard> listShards(final String streamName) throws TransientKinesisException {
  return wrapExceptions(new Callable<List<Shard>>() {

    @Override
    public List<Shard> call() throws Exception {
      List<Shard> shards = Lists.newArrayList();
      String lastShardId = null;

      StreamDescription description;
      do {
        description = kinesis.describeStream(streamName, lastShardId)
            .getStreamDescription();

        shards.addAll(description.getShards());
        lastShardId = shards.get(shards.size() - 1).getShardId();
      } while (description.getHasMoreShards());

      return shards;
    }
  });
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void shouldListAllShards() throws Exception {
  Shard shard1 = new Shard().withShardId(SHARD_1);
  Shard shard2 = new Shard().withShardId(SHARD_2);
  Shard shard3 = new Shard().withShardId(SHARD_3);
  given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult()
      .withStreamDescription(new StreamDescription()
          .withShards(shard1, shard2)
          .withHasMoreShards(true)));
  given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult()
      .withStreamDescription(new StreamDescription()
          .withShards(shard3)
          .withHasMoreShards(false)));

  List<Shard> shards = underTest.listShards(STREAM);

  assertThat(shards).containsOnly(shard1, shard2, shard3);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Override
public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
  int nextShardId = 0;
  if (exclusiveStartShardId != null) {
    nextShardId = parseInt(exclusiveStartShardId) + 1;
  }
  boolean hasMoreShards = nextShardId + 1 < shardedData.size();

  List<Shard> shards = new ArrayList<>();
  if (nextShardId < shardedData.size()) {
    shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
  }

  HttpResponse response = new HttpResponse(null, null);
  response.setStatusCode(200);
  DescribeStreamResult result = new DescribeStreamResult();
  result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
  result.withStreamDescription(
      new StreamDescription()
          .withHasMoreShards(hasMoreShards)
          .withShards(shards)
          .withStreamName(streamName));
  return result;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
/**
 * Utility function to convert {@link StreamShardMetadata} into {@link StreamShardHandle}.
 *
 * @param streamShardMetadata the {@link StreamShardMetadata} to be converted
 * @return a {@link StreamShardHandle} object
 */
public static StreamShardHandle convertToStreamShardHandle(StreamShardMetadata streamShardMetadata) {
	Shard shard = new Shard();
	shard.withShardId(streamShardMetadata.getShardId());
	shard.withParentShardId(streamShardMetadata.getParentShardId());
	shard.withAdjacentParentShardId(streamShardMetadata.getAdjacentParentShardId());

	HashKeyRange hashKeyRange = new HashKeyRange();
	hashKeyRange.withStartingHashKey(streamShardMetadata.getStartingHashKey());
	hashKeyRange.withEndingHashKey(streamShardMetadata.getEndingHashKey());
	shard.withHashKeyRange(hashKeyRange);

	SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
	sequenceNumberRange.withStartingSequenceNumber(streamShardMetadata.getStartingSequenceNumber());
	sequenceNumberRange.withEndingSequenceNumber(streamShardMetadata.getEndingSequenceNumber());
	shard.withSequenceNumberRange(sequenceNumberRange);

	return new StreamShardHandle(streamShardMetadata.getStreamName(), shard);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<StreamShardHandle> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
	List<StreamShardHandle> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);

		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new StreamShardHandle(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public NonReshardedStreamsKinesis(Map<String, Integer> streamsToShardCount) {
	for (Map.Entry<String, Integer> streamToShardCount : streamsToShardCount.entrySet()) {
		String streamName = streamToShardCount.getKey();
		int shardCount = streamToShardCount.getValue();

		if (shardCount == 0) {
			// don't do anything
		} else {
			List<StreamShardHandle> shardsOfStream = new ArrayList<>(shardCount);
			for (int i = 0; i < shardCount; i++) {
				shardsOfStream.add(
					new StreamShardHandle(
						streamName,
						new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(i))));
			}
			streamsWithListOfShards.put(streamName, shardsOfStream);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<KinesisStreamShard> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
	List<KinesisStreamShard> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);

		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new KinesisStreamShard(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public NonReshardedStreamsKinesis(Map<String,Integer> streamsToShardCount) {
	for (Map.Entry<String,Integer> streamToShardCount : streamsToShardCount.entrySet()) {
		String streamName = streamToShardCount.getKey();
		int shardCount = streamToShardCount.getValue();

		if (shardCount == 0) {
			// don't do anything
		} else {
			List<KinesisStreamShard> shardsOfStream = new ArrayList<>(shardCount);
			for (int i=0; i < shardCount; i++) {
				shardsOfStream.add(
					new KinesisStreamShard(
						streamName,
						new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(i))));
			}
			streamsWithListOfShards.put(streamName, shardsOfStream);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Before
public void setup() throws Exception {
    KinesisEndpoint endpoint = new KinesisEndpoint(null, "streamName", component);
    endpoint.setAmazonKinesisClient(kinesisClient);
    endpoint.setIteratorType(ShardIteratorType.LATEST);
    undertest = new KinesisConsumer(endpoint, processor);

    when(kinesisClient.getRecords(any(GetRecordsRequest.class)))
        .thenReturn(new GetRecordsResult()
            .withNextShardIterator("nextShardIterator")
        );
    when(kinesisClient.describeStream(any(DescribeStreamRequest.class)))
        .thenReturn(new DescribeStreamResult()
            .withStreamDescription(new StreamDescription()
                .withShards(new Shard().withShardId("shardId"))
            )
        );
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class)))
        .thenReturn(new GetShardIteratorResult()
            .withShardIterator("shardIterator")
        );
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout)
{
    KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout);
    KinesisTableHandle kinesisTableHandle = kinesislayout.getTable();

    InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName());

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    for (Shard shard : desc.getShards()) {
        KinesisSplit split = new KinesisSplit(connectorId,
                kinesisTableHandle.getStreamName(),
                kinesisTableHandle.getMessageDataFormat(),
                shard.getShardId(),
                shard.getSequenceNumberRange().getStartingSequenceNumber(),
                shard.getSequenceNumberRange().getEndingSequenceNumber());
        builder.add(split);
    }

    return new FixedSplitSource(builder.build());
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<Shard> describeStream(String stream) {
	AmazonKinesisAsync amazonKinesis = localKinesisResource.getResource();

	String exclusiveStartShardId = null;

	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
			.withStreamName(stream);

	List<Shard> shardList = new ArrayList<>();

	while (true) {
		DescribeStreamResult describeStreamResult = null;

		describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
		describeStreamResult = amazonKinesis.describeStream(describeStreamRequest);
		StreamDescription streamDescription = describeStreamResult.getStreamDescription();
		if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
			shardList.addAll(streamDescription.getShards());

			if (streamDescription.getHasMoreShards()) {
				exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
				continue;
			}
			else {
				return shardList;
			}
		}
		try {
			Thread.sleep(100);
		}
		catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			throw new IllegalStateException(e);
		}
	}
}