Java InitialPositionInStream.LATEST属性代码示例

/**
 * Get KCL config for a given system stream.
 * @param system name of the system
 * @param stream name of the stream
 * @param appName name of the application
 * @return Stream scoped KCL configs required to build
 *         {@link KinesisClientLibConfiguration}
 */
public KinesisClientLibConfiguration getKinesisClientLibConfig(String system, String stream, String appName) {
  ClientConfiguration clientConfig = getAWSClientConfig(system);
  String workerId = appName + "-" + UUID.randomUUID();
  InitialPositionInStream startPos = InitialPositionInStream.LATEST;
  AWSCredentialsProvider provider = credentialsProviderForStream(system, stream);
  KinesisClientLibConfiguration kinesisClientLibConfiguration =
      new KinesisClientLibConfiguration(appName, stream, provider, workerId)
          .withRegionName(getRegion(system, stream).getName())
          .withKinesisClientConfig(clientConfig)
          .withCloudWatchClientConfig(clientConfig)
          .withDynamoDBClientConfig(clientConfig)
          .withInitialPositionInStream(startPos)
          .withCallProcessRecordsEvenForEmptyRecordList(true); // For health monitoring metrics.
  // First, get system scoped configs for KCL and override with configs set at stream scope.
  setKinesisClientLibConfigs(
      subset(String.format(CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, system)), kinesisClientLibConfiguration);
  setKinesisClientLibConfigs(subset(String.format(CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, system, stream)),
      kinesisClientLibConfiguration);
  return kinesisClientLibConfiguration;
}
 

@Test
public void shouldMapAllShardsToCheckpoints() throws Exception {
  given(shard1.getShardId()).willReturn("shard-01");
  given(shard2.getShardId()).willReturn("shard-02");
  given(shard3.getShardId()).willReturn("shard-03");
  given(kinesisClient.listShards("stream")).willReturn(asList(shard1, shard2, shard3));

  StartingPoint startingPoint = new StartingPoint(InitialPositionInStream.LATEST);
  DynamicCheckpointGenerator underTest = new DynamicCheckpointGenerator("stream",
      startingPoint);

  KinesisReaderCheckpoint checkpoint = underTest.generate(kinesisClient);

  assertThat(checkpoint).hasSize(3);
}
 

public void start() {
    final JavaStreamingContext context = new JavaStreamingContext(conf, checkpointInterval);

    // for graceful shutdown of the application ...
    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            System.out.println("Shutting down streaming app...");
            context.stop(true, true);
            System.out.println("Shutdown of streaming app complete.");
        }
    });

    JKinesisReceiver receiver = new JKinesisReceiver(appName, streamName,
                                                     endpointUrl, regionName,
                                                     checkpointInterval,
                                                     InitialPositionInStream.LATEST);

    JavaDStream<String> dstream = context.receiverStream(receiver);

    JavaDStream<EventRecord> recs = dstream.map(new EventRecordMapFunc());

    recs.print();

    // persist to DStream to Cassandra
    javaFunctions(recs)
        .writerBuilder("canary", "eventrecord", mapToRow(EventRecord.class))
        .saveToCassandra();


    System.out.println("Start Spark Stream Processing...");

    context.start();
    context.awaitTermination();

}
 

private InitialPositionInStream convertToPosition(KinesisInputProperties.OffsetType offsetType) {
    switch (offsetType) {
    case LATEST:
        return InitialPositionInStream.LATEST;
    case EARLIEST:
        return InitialPositionInStream.TRIM_HORIZON;
    default:
        TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow(
                String.format("Do not support OffsetType %s", offsetType));
        return null;
    }
}
 

public void configure() throws Exception {
    if (!isConfigured) {
        validateConfig();

        try {
            String userAgent = "AWSKinesisManagedConsumer/" + this.version;

            if (this.positionInStream != null) {
                streamPosition = InitialPositionInStream.valueOf(this.positionInStream);
            } else {
                streamPosition = InitialPositionInStream.LATEST;
            }

            // append the environment name to the application name
            if (environmentName != null) {
                appName = String.format("%s-%s", appName, environmentName);
            }

            // ensure the JVM will refresh the cached IP values of AWS
            // resources
            // (e.g. service endpoints).
            java.security.Security.setProperty("networkaddress.cache.ttl", "60");

            String workerId = NetworkInterface.getNetworkInterfaces() + ":" + UUID.randomUUID();
            LOG.info("Using Worker ID: " + workerId);

            // obtain credentials using the default provider chain or the
            // credentials provider supplied
            AWSCredentialsProvider credentialsProvider = this.credentialsProvider == null ? new DefaultAWSCredentialsProviderChain()
                    : this.credentialsProvider;

            LOG.info("Using credentials with Access Key ID: "
                    + credentialsProvider.getCredentials().getAWSAccessKeyId());

            config = new KinesisClientLibConfiguration(appName, streamName,
                    credentialsProvider, workerId).withInitialPositionInStream(streamPosition).withKinesisEndpoint(
                    kinesisEndpoint);

            config.getKinesisClientConfiguration().setUserAgent(userAgent);

            if (regionName != null) {
                Region region = Region.getRegion(Regions.fromName(regionName));
                config.withRegionName(region.getName());
            }

            if (this.maxRecords != -1)
                config.withMaxRecords(maxRecords);

            if (this.positionInStream != null)
                config.withInitialPositionInStream(InitialPositionInStream.valueOf(this.positionInStream));

            LOG.info(String.format(
                    "Amazon Kinesis Managed Client prepared for %s on %s in %s (%s) using %s Max Records",
                    config.getApplicationName(), config.getStreamName(),
                    config.getRegionName(), config.getWorkerIdentifier(),
                    config.getMaxRecords()));

            isConfigured = true;
        } catch (Exception e) {
            throw new InvalidConfigurationException(e);
        }
    }
}
 

public void configure() throws Exception {
	if (!isConfigured) {
		validateConfig();

		if (this.positionInStream != null) {
			streamPosition = InitialPositionInStream
					.valueOf(this.positionInStream);
		} else {
			streamPosition = InitialPositionInStream.LATEST;
		}

		// append the environment name to the application name
		if (environmentName != null) {
			appName = String.format("%s-%s", appName, environmentName);
		}

		// ensure the JVM will refresh the cached IP values of AWS resources
		// (e.g. service endpoints).
		java.security.Security
				.setProperty("networkaddress.cache.ttl", "60");

		String workerId = NetworkInterface.getNetworkInterfaces() + ":"
				+ UUID.randomUUID();
		LOG.info("Using Worker ID: " + workerId);

		// obtain credentials using the default provider chain or the
		// credentials provider supplied
		AWSCredentialsProvider credentialsProvider = this.credentialsProvider == null ? new DefaultAWSCredentialsProviderChain()
				: this.credentialsProvider;

		LOG.info("Using credentials with Access Key ID: "
				+ credentialsProvider.getCredentials().getAWSAccessKeyId());

		config = new KinesisClientLibConfiguration(appName, streamName,
				credentialsProvider, workerId).withInitialPositionInStream(
				streamPosition).withKinesisEndpoint(kinesisEndpoint);

		config.getKinesisClientConfiguration().setUserAgent(
				StreamAggregator.AWSApplication);

		if (regionName != null) {
			Region region = Region.getRegion(Regions.fromName(regionName));
			config.withRegionName(region.getName());
		}

		if (maxRecords != -1)
			config.withMaxRecords(maxRecords);

		// initialise the Aggregators
		aggGroup = buildAggregatorsFromConfig();

		LOG.info(String
				.format("Amazon Kinesis Aggregators Managed Client prepared for %s on %s in %s (%s) using %s Max Records",
						config.getApplicationName(),
						config.getStreamName(), config.getRegionName(),
						config.getWorkerIdentifier(),
						config.getMaxRecords()));

		isConfigured = true;
	}
}