Java DescribeStreamRequest.setExclusiveStartShardId方法代码示例

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入方法依赖的package包/类
private void refreshShards() {
  DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
  describeStreamRequest.setStreamName(streamName);
  String exclusiveStartShardId = null;
  List<Shard> shards = new ArrayList<>();

  do {
    describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
    DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
    shards.addAll(describeStreamResult.getStreamDescription().getShards());

    if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
      exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
    } else {
      exclusiveStartShardId = null;
    }
  } while (exclusiveStartShardId != null);

  this.shards = shards;
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入方法依赖的package包/类
/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 *
 * Gets from the internal cache unless not yet created or too old.
 *
 * @param streamName
 * @return
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription desc = this.streamMap.get(streamName);
    if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        desc = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            desc.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        } while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, desc);
    }

    return desc;
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入方法依赖的package包/类
/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This
 * jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result)
 * @return the result of the describe stream operation
 */
private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
				describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
			LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for "
				+ backoffMillis + " millis.");
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current " +
				"describeStream operation will not contain any shard information.");
		}
	}

	// Kinesalite (mock implementation of Kinesis) does not correctly exclude shards before the exclusive
	// start shard id in the returned shards list; check if we need to remove these erroneously returned shards
	if (startShardId != null) {
		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		Iterator<Shard> shardItr = shards.iterator();
		while (shardItr.hasNext()) {
			if (StreamShardHandle.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) {
				shardItr.remove();
			}
		}
	}

	return describeStreamResult;
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入方法依赖的package包/类
/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess.
 *
 * This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This
 * jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result)
 * @return the result of the describe stream operation
 */
private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
				describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
			LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for "
				+ backoffMillis + " millis.");
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current " +
				"describeStream operation will not contain any shard information.");
		}
	}

	// Kinesalite (mock implementation of Kinesis) does not correctly exclude shards before the exclusive
	// start shard id in the returned shards list; check if we need to remove these erroneously returned shards
	if (startShardId != null) {
		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		Iterator<Shard> shardItr = shards.iterator();
		while (shardItr.hasNext()) {
			if (KinesisStreamShard.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) {
				shardItr.remove();
			}
		}
	}

	return describeStreamResult;
}
 

import com.amazonaws.services.kinesis.model.DescribeStreamRequest; //导入方法依赖的package包/类
public static void main(String[] args) {
	AmazonKinesisClient kinesisClient = Helper.setupKinesisClient();

	// Retrieve the Shards from a Stream
	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(Helper.properties().getProperty("kinesisStreamName"));
	DescribeStreamResult describeStreamResult;
	List<Shard> shards = new ArrayList<>();
	String lastShardId = null;

	do {
	    describeStreamRequest.setExclusiveStartShardId(lastShardId);
	    describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
	    shards.addAll(describeStreamResult.getStreamDescription().getShards());
	    if (shards.size() > 0) {
	        lastShardId = shards.get(shards.size() - 1).getShardId();
	    }
	} while (describeStreamResult.getStreamDescription().getHasMoreShards());

	// Get Data from the Shards in a Stream
	// Hard-coded to use only 1 shard
	String shardIterator;
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
	getShardIteratorRequest.setStreamName(Helper.properties().getProperty("kinesisStreamName"));
	getShardIteratorRequest.setShardId(shards.get(0).getShardId());
	getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");

	GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
	shardIterator = getShardIteratorResult.getShardIterator();

	// Continuously read data records from shard.
	List<Record> records;
	while (true) {
		// Create new GetRecordsRequest with existing shardIterator.
		// Set maximum records to return to 1000.
		GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
		getRecordsRequest.setShardIterator(shardIterator);
		getRecordsRequest.setLimit(1000);

		GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest);

		// Put result into record list. Result may be empty.
		records = result.getRecords();

		// Print records
		for (Record record : records) {
			ByteBuffer byteBuffer = record.getData();
			System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(),
					new String(byteBuffer.array())));
		}

		try {
			Thread.sleep(1000);
		} catch (InterruptedException exception) {
			throw new RuntimeException(exception);
		}

		shardIterator = result.getNextShardIterator();
	}
}