Java ParameterTool.getInt方法代码示例

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
@Override
public void configure(ParameterTool parameterTool) {
	if (!parameterTool.has("iterations") && !parameterTool.has("convergence_threshold")) {
		// no configuration so use default iterations and maximum threshold
		value.iterations = defaultIterations;
		value.convergenceThreshold = Double.MAX_VALUE;
	} else {
		// use configured values and maximum default for unset values
		value.iterations = parameterTool.getInt("iterations", Integer.MAX_VALUE);
		Util.checkParameter(value.iterations > 0,
			"iterations must be greater than zero");

		value.convergenceThreshold = parameterTool.getDouble("convergence_threshold", Double.MAX_VALUE);
		Util.checkParameter(value.convergenceThreshold > 0,
			"convergence threshold must be greater than zero");
	}
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public static void main(String[] args) {
	try {
		// startup checks and logging
		EnvironmentInformation.logEnvironmentInfo(LOG, "ZooKeeper Quorum Peer", args);
		
		final ParameterTool params = ParameterTool.fromArgs(args);
		final String zkConfigFile = params.getRequired("zkConfigFile");
		final int peerId = params.getInt("peerId");

		// Run quorum peer
		runFlinkZkQuorumPeer(zkConfigFile, peerId);
	}
	catch (Throwable t) {
		LOG.error("Error running ZooKeeper quorum peer: " + t.getMessage(), t);
		System.exit(-1);
	}
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
private void parseConfigurations(String[] args) {

		LOG.info("ApplicationMain Main.. Arguments: {}", Arrays.asList(args));

		ParameterTool parameterTool = ParameterTool.fromArgs(args);
		LOG.info("Parameter Tool: {}", parameterTool.toMap());

		if(parameterTool.getNumberOfParameters() != 2) {
			printUsage();
			System.exit(1);
		}

		String configDirPath = parameterTool.getRequired("configDir");
		try {
			byte[] configurationData = Files.readAllBytes(Paths.get(configDirPath + File.separator + configFile));
			String jsonData = new String(configurationData);
			LOG.info("App Configurations raw data: {}", jsonData);
			Gson gson = new Gson();
			appConfiguration = gson.fromJson(jsonData, AppConfiguration.class);
		} catch (IOException e) {
			LOG.error("Could not read {}",configFile, e);
			System.exit(1);
		}

		runMode = parameterTool.getInt("mode");

		pravega = new FlinkPravegaParams(ParameterTool.fromArgs(args));
	}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public SummarizationJobParameters(ParameterTool params) {
    timelyHostname = params.getRequired("timelyHostname");
    timelyTcpPort = params.getInt("timelyTcpPort", 4241);
    timelyHttpsPort = params.getInt("timelyHttpsPort", 4242);
    timelyWssPort = params.getInt("timelyWssPort", 4243);
    doLogin = params.getBoolean("doLogin", false);
    timelyUsername = params.get("timelyUsername", null);
    timelyPassword = params.get("timelyPassword", null);
    keyStoreFile = params.getRequired("keyStoreFile");
    keyStoreType = params.get("keyStoreType", "JKS");
    keyStorePass = params.getRequired("keyStorePass");
    trustStoreFile = params.getRequired("trustStoreFile");
    trustStoreType = params.get("trustStoreType", "JKS");
    trustStorePass = params.getRequired("trustStorePass");
    hostVerificationEnabled = params.getBoolean("hostVerificationEnabled", true);
    bufferSize = params.getInt("bufferSize", 10485760);
    String metricNames = params.getRequired("metrics");
    if (null != metricNames) {
        metrics = metricNames.split(",");
    } else {
        metrics = null;
    }
    startTime = params.getLong("startTime", 0L);
    endTime = params.getLong("endTime", 0L);
    interval = params.getRequired("interval");
    intervalUnits = params.getRequired("intervalUnits");
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
					"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
					"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
					"type the input text into the command line");
			return;
		}

		DataServiceFacade dataService = new DataServiceFacade(DataEntityType.WORD_COUNT);

		dataService.setUpEmbeddedCassandra();
		dataService.setUpDataModel();

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordCount> result = text

				.flatMap(new FlatMapFunction<String, WordCount>() {
					@Override
					public void flatMap(String value, Collector<WordCount> out) {
						// normalize and split the line
						String[] words = value.toLowerCase().split("\\s");

						// emit the pairs
						for (String word : words) {
							if (!word.isEmpty()) {
								//Do not accept empty word, since word is defined as primary key in C* table
								out.collect(new WordCount(word, 1L));
							}
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordCount>() {
					@Override
					public WordCount reduce(WordCount a, WordCount b) {
						return new WordCount(a.getWord(), a.getCount() + b.getCount());
					}
				});

		CassandraSink.addSink(result)
				.setHost("127.0.0.1")
				.build();

		CQLPrintSinkFunction<WordCount, WordCount> func = new CQLPrintSinkFunction();
		func.setDataModel(dataService, 10);
		result.addSink(func).setParallelism(1);

		env.execute("Socket Window WordCount (POJO) w/ C* Sink");
	}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
					"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
					"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
					"type the input text into the command line");
			return;
		}

		DataServiceFacade dataService = new DataServiceFacade(DataEntityType.WORD_COUNT);

		dataService.setUpEmbeddedCassandra();
		dataService.setUpDataModel();

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<Tuple2<String, Long>> result = text

				.flatMap(new FlatMapFunction<String, Tuple2<String, Long>>() {
					@Override
					public void flatMap(String value, Collector<Tuple2<String, Long>> out) {
						// normalize and split the line
						String[] words = value.toLowerCase().split("\\s");

						// emit the pairs
						for (String word : words) {
							//Do not accept empty word, since word is defined as primary key in C* table
							if (!word.isEmpty()) {
								out.collect(new Tuple2<String, Long>(word, 1L));
							}
						}
					}
				})

				.keyBy(0)
				.timeWindow(Time.seconds(5))
				.sum(1)
				;

		CassandraSink.addSink(result)
				.setQuery("INSERT INTO " + WordCount.CQL_KEYSPACE_NAME + "." + WordCount.CQL_TABLE_NAME + "(word, count) " +
						"values (?, ?);")
				.setHost("127.0.0.1")
				.build();

		CQLPrintSinkFunction<Tuple2<String, Long>, WordCount> func = new CQLPrintSinkFunction();
		func.setDataModel(dataService, 10);
		result.addSink(func).setParallelism(1);

		env.execute("Socket Window WordCount (Tuple) w/ C* Sink");
	}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {

  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final int port = Integer.valueOf(parameter.getRequired("port"));
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final long windowSize = parameter.getLong("windowSize", 10);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "SECONDS"));
  final int parallelism = parameter.getInt("parallelism", 1);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Port: " + port);
  System.out.println("Output: " + outputPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<String> text = env.socketTextStream("localhost", port, "\n");

  DataStream<WordWithCount> windowCounts = text
      .flatMap(new WordTokenizer())
      .keyBy("word")
      .timeWindow(Time.of(windowSize, windowUnit))
      .reduce(new WordCountReducer())
      .setParallelism(parallelism);

  windowCounts.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE);

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {
  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final int port = Integer.valueOf(parameter.getRequired("port"));
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final long windowSize = parameter.getLong("windowSize", 10);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "SECONDS"));
  final int rankSize = parameter.getInt("rankSize", 3);
  final int parallelism = parameter.getInt("parallelism", 1);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Port: " + port);
  System.out.println("Output: " + outputPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Rank Size: " + rankSize);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<TimedWord> timedWords = env.addSource(new StoppableTimedWordSocketSource("localhost", port))
      .assignTimestampsAndWatermarks(new EventTimestampExtractor());

  DataStream<WindowWordWithCount> windowCounts = timedWords
      .keyBy(new WordKeySelector())
      .timeWindow(Time.of(windowSize, windowUnit))
      .aggregate(new TimedWordCounterAggregator(), new TimedWordCounterWindowFunction())
      .setParallelism(parallelism);

  DataStream<WindowWordRanking> ranking = windowCounts.timeWindowAll(Time.of(windowSize, windowUnit))
      .apply(new WordRankerWindowFunction(rankSize));

  ranking.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE);

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {
  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final String kafkaZookeeper = parameter.get("kafka.zookeeper", "localhost:2181");
  final String kafkaBootstrap = parameter.get("kafka.bootstrap", "localhost:9092");
  final String kafkaTopic = parameter.get("kafka.topic", "topic-query-3");
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final String elasticsearch = parameter.get("elasticsearch", null);
  final long windowSize = parameter.getLong("windowSize", 10);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "SECONDS"));
  final int rankSize = parameter.getInt("rankSize", 3);
  final long tsEnd = parameter.getLong("tsEnd", 100000L);
  final Set<String> ignoredWords = Sets.newHashSet(parameter.get("ignoredWords", "")
      .trim().split(","));
  final int parallelism = parameter.getInt("parallelism", 1);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  env.setParallelism(parallelism);
  final KafkaProperties kafkaProps = new KafkaProperties(kafkaBootstrap, kafkaZookeeper);
  final ESProperties elasticsearchProps = ESProperties.fromPropString(elasticsearch);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Kafka Zookeeper: " + kafkaZookeeper);
  System.out.println("Kafka Bootstrap: " + kafkaBootstrap);
  System.out.println("Kafka Topic: " + kafkaTopic);
  System.out.println("Output: " + outputPath);
  System.out.println("Elasticsearch: " + elasticsearch);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Rank Size: " + rankSize);
  System.out.println("Timestamp End: " + tsEnd);
  System.out.println("Ignored Words: " + ignoredWords);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<TimedWord> timedWords = env.addSource(new StoppableTimedWordKafkaSource(kafkaTopic, kafkaProps, tsEnd));

  DataStream<TimedWord> fileterTimedWords = timedWords.filter(new TimedWordFilter(ignoredWords))
      .assignTimestampsAndWatermarks(new EventTimestampExtractor());

  DataStream<WindowWordWithCount> windowCounts = fileterTimedWords
      .keyBy(new WordKeySelector())
      .timeWindow(Time.of(windowSize, windowUnit))
      .aggregate(new TimedWordCounterAggregator(), new TimedWordCounterWindowFunction());

  DataStream<WindowWordRanking> ranking = windowCounts.timeWindowAll(Time.of(windowSize, windowUnit))
      .apply(new WordRankerWindowFunction(rankSize));

  ranking.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE);

  if (elasticsearch != null) {
    ranking.addSink(new ESSink<>(elasticsearchProps,
        new MyESSinkFunction(elasticsearchProps.getIndexName(), elasticsearchProps.getTypeName()))
    );
  }

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {

  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final String kafkaZookeeper = parameter.get("kafka.zookeeper", "localhost:2181");
  final String kafkaBootstrap = parameter.get("kafka.bootstrap", "localhost:9092");
  final String kafkaTopic = parameter.get("kafka.topic", "socstream");
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final String elasticsearch = parameter.get("elasticsearch", null);
  final Path metadataPath = FileSystems.getDefault().getPath(parameter.get("metadata", "./metadata.yml"));
  final long windowSize = parameter.getLong("windowSize", 70);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "MINUTES"));
  final long matchStart = parameter.getLong("match.start", 10753295594424116L);
  final long matchEnd = parameter.getLong("match.end", 14879639146403495L);
  final long matchIntervalStart = parameter.getLong("match.interval.start", 12557295594424116L);
  final long matchIntervalEnd = parameter.getLong("match.interval.end", 13086639146403495L);
  final int parallelism = parameter.getInt("parallelism", 1);
  final Match match = MatchService.fromYamlFile(metadataPath);
  final Set<Long> ignoredSensors = MatchService.collectIgnoredSensors(match);
  final Map<Long,Long> sid2Pid = MatchService.collectSid2Pid(match);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  final KafkaProperties kafkaProps = new KafkaProperties(kafkaBootstrap);
  final ESProperties elasticsearchProps = ESProperties.fromPropString(elasticsearch);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Kafka Zookeeper: " + kafkaZookeeper);
  System.out.println("Kafka Bootstrap: " + kafkaBootstrap);
  System.out.println("Kafka Topic: " + kafkaTopic);
  System.out.println("Output: " + outputPath);
  System.out.println("Elasticsearch: " + elasticsearch);
  System.out.println("Metadata: " + metadataPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Match Start: " + matchStart);
  System.out.println("Match End: " + matchEnd);
  System.out.println("Match Interval Start: " + matchIntervalStart);
  System.out.println("Match Interval End: " + matchIntervalEnd);
  System.out.println("Ignored Sensors: " + ignoredSensors);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<RichSensorEvent> sensorEvents = env.addSource(
      new RichSensorEventKafkaSource(kafkaTopic, kafkaProps, matchStart, matchEnd,
          matchIntervalStart, matchIntervalEnd, ignoredSensors, sid2Pid
      )
  ).assignTimestampsAndWatermarks(new RichSensorEventTimestampExtractor()).setParallelism(parallelism);

  DataStream<PlayerRunningStatistics> statistics = sensorEvents.keyBy(new RichSensorEventKeyer())
      .timeWindow(Time.of(windowSize, windowUnit))
      .aggregate(new PlayerRunningStatisticsCalculatorAggregator(), new PlayerRunningStatisticsCalculatorWindowFunction())
      .setParallelism(parallelism);

  statistics.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE).setParallelism(1);

  if (elasticsearch != null) {
    statistics.addSink(new ESSink<>(elasticsearchProps,
        new PlayerRunningStatisticsESSinkFunction(elasticsearchProps.getIndexName(), elasticsearchProps.getTypeName()))
    ).setParallelism(1);
  }

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {

  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final String kafkaZookeeper = parameter.get("kafka.zookeeper", "localhost:2181");
  final String kafkaBootstrap = parameter.get("kafka.bootstrap", "localhost:9092");
  final String kafkaTopic = parameter.get("kafka.topic", "socstream");
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final String elasticsearch = parameter.get("elasticsearch", null);
  final Path metadataPath = FileSystems.getDefault().getPath(parameter.get("metadata", "./metadata.yml"));
  final long windowSize = parameter.getLong("windowSize", 70);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "MINUTES"));
  final int rankSize = parameter.getInt("rankSize", 5);
  final long matchStart = parameter.getLong("match.start", 10753295594424116L);
  final long matchEnd = parameter.getLong("match.end", 14879639146403495L);
  final long matchIntervalStart = parameter.getLong("match.interval.start", 12557295594424116L);
  final long matchIntervalEnd = parameter.getLong("match.interval.end", 13086639146403495L);
  final int parallelism = parameter.getInt("parallelism", 1);

  final Match match = MatchService.fromYamlFile(metadataPath);
  final Set<Long> ignoredSensors = MatchService.collectIgnoredSensors(match);
  final Map<Long,Long> sid2Pid = MatchService.collectSid2Pid(match);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  final KafkaProperties kafkaProps = new KafkaProperties(kafkaBootstrap);
  final ESProperties elasticsearchProps = ESProperties.fromPropString(elasticsearch);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Kafka Zookeeper: " + kafkaZookeeper);
  System.out.println("Kafka Bootstrap: " + kafkaBootstrap);
  System.out.println("Kafka Topic: " + kafkaTopic);
  System.out.println("Output: " + outputPath);
  System.out.println("Elasticsearch: " + elasticsearch);
  System.out.println("Metadata: " + metadataPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Rank Size: " + rankSize);
  System.out.println("Match Start: " + matchStart);
  System.out.println("Match End: " + matchEnd);
  System.out.println("Match Interval Start: " + matchIntervalStart);
  System.out.println("Match Interval End: " + matchIntervalEnd);
  System.out.println("Ignored Sensors: " + ignoredSensors);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<SpeedSensorEvent> sensorEvents = env.addSource(
      new SpeedSensorEventKafkaSource(kafkaTopic, kafkaProps, matchStart, matchEnd,
          matchIntervalStart, matchIntervalEnd, ignoredSensors, sid2Pid
      )
  ).assignTimestampsAndWatermarks(new SpeedSensorEventTimestampExtractor()).setParallelism(parallelism);

  DataStream<PlayerSpeedStatistics> statistics = sensorEvents.keyBy(new SpeedSensorEventKeyer())
      .timeWindow(Time.of(windowSize, windowUnit))
      .aggregate(new PlayerSpeedStatisticsCalculatorAggregator(), new PlayerSpeedStatisticsCalculatorWindowFunction())
      .setParallelism(parallelism);

  DataStream<PlayersSpeedRanking> ranking = statistics.timeWindowAll(Time.of(windowSize, windowUnit))
      .apply(new GlobalRankerWindowFunction(rankSize));

  ranking.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE).setParallelism(1);

  if (elasticsearch != null) {
    ranking.addSink(new ESSink<>(elasticsearchProps,
        new PlayerSpeedRankingESSinkFunction(elasticsearchProps.getIndexName(), elasticsearchProps.getTypeName()))
    ).setParallelism(1);
  }

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {

  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final String kafkaZookeeper = parameter.get("kafka.zookeeper", "localhost:2181");
  final String kafkaBootstrap = parameter.get("kafka.bootstrap", "localhost:9092");
  final String kafkaTopic = parameter.get("kafka.topic", "socstream");
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final String elasticsearch = parameter.get("elasticsearch", null);
  final Path metadataPath = FileSystems.getDefault().getPath(parameter.get("metadata", "./metadata.yml"));
  final long windowSize = parameter.getLong("windowSize", 70);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "MINUTES"));
  final long matchStart = parameter.getLong("match.start", 10753295594424116L);
  final long matchEnd = parameter.getLong("match.end", 14879639146403495L);
  final long matchIntervalStart = parameter.getLong("match.interval.start", 12557295594424116L);
  final long matchIntervalEnd = parameter.getLong("match.interval.end", 13086639146403495L);
  final int parallelism = parameter.getInt("parallelism", 1);

  final Match match = MatchService.fromYamlFile(metadataPath);
  final Set<Long> ignoredSensors = MatchService.collectIgnoredSensors(match);
  final Map<Long,Long> sid2Pid = MatchService.collectSid2Pid(match);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  final KafkaProperties kafkaProps = new KafkaProperties(kafkaBootstrap);
  final ESProperties elasticsearchProps = ESProperties.fromPropString(elasticsearch);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Kafka Zookeeper: " + kafkaZookeeper);
  System.out.println("Kafka Bootstrap: " + kafkaBootstrap);
  System.out.println("Kafka Topic: " + kafkaTopic);
  System.out.println("Output: " + outputPath);
  System.out.println("Elasticsearch: " + elasticsearch);
  System.out.println("Metadata: " + metadataPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Match Start: " + matchStart);
  System.out.println("Match End: " + matchEnd);
  System.out.println("Match Interval Start: " + matchIntervalStart);
  System.out.println("Match Interval End: " + matchIntervalEnd);
  System.out.println("Ignored Sensors: " + ignoredSensors);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<PositionSensorEvent> sensorEvents = env.addSource(
      new PositionSensorEventKafkaSource(kafkaTopic, kafkaProps, matchStart, matchEnd,
          matchIntervalStart, matchIntervalEnd, ignoredSensors, sid2Pid
      ).assignTimestampsAndWatermarks(new PositionSensorEventTimestampExtractor())).setParallelism(1);

  DataStream<PlayerGridStatistics> statistics = sensorEvents.keyBy(new PositionSensorEventKeyer())
      .timeWindow(Time.of(windowSize, windowUnit))
      .aggregate(new PlayerOnGridStatisticsCalculatorAggregator(), new PlayerOnGridStatisticsCalculatorWindowFunction())
      .setParallelism(parallelism);

  statistics.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE).setParallelism(1);

  if (elasticsearch != null) {
    statistics.addSink(new ESSink<>(elasticsearchProps,
        new PlayerGridStatisticsESSinkFunction(elasticsearchProps.getIndexName(), elasticsearchProps.getTypeName()))
    ).setParallelism(1);
  }

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
/**
 * org.apache.flink.streaming.api.functions.demo
 *
 * @param args 参数样例：
 *             --topic data --cgroup data_group
 *             --sourceParallelism 2
 *             --hdfsSinkParallelism 10
 *             --sinkParallelism  10
 *             --mapParallelism 10
 *             --keybyms 60000
 *             --charset UTF-8
 *             --confdir /home/flow/DAT/temp
 * @throws Exception
 */
public static void main(String... args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // 解析参数

    final ParameterTool parameterTool = ParameterTool.fromArgs(args);
    int argLength = 7;
    if (parameterTool.getNumberOfParameters() < argLength) {
        System.out.println("Missing parameters!");
        System.out.println("\nUsage: GlobleConfig  as ExpressStatusFlow\n" +
                "--topic <topic> \n" +
                "--cgroup <consumerGroupName> \n" +
                "--sourceParallelism <sourceParallelism default 1> \n" +
                "--sinkParallelism <sinkParallelism default 1> \n" +
                "--mapParallelism <mapParallelism default 1> \n" +
                "--hdfsSinkParallelism <hdfsSinkParallelism default 1> \n" +
                "--keybyms <keybyms default 100> \n" +
                "--charset <CharsetName Messsage> \n" +
                "--confdir <config file path for All NODE > \n"
        );
        return;
    }

    String topic = parameterTool.get("topic");
    int mapParallelism = parameterTool.getInt("mapParallelism");
    int sourceParallelism = parameterTool.getInt("sourceParallelism");
    int sinkParallelism = parameterTool.getInt("sinkParallelism");
    int hdfsSinkParallelism = parameterTool.getInt("hdfsSinkParallelism");
    int keyByMilliseconds = parameterTool.getInt("keybyms");

    env.getConfig().enableSysoutLogging();
    Configuration conf = new Configuration();
    conf.setString("topic", topic);
    conf.setString("charset", parameterTool.get("charset"));
    conf.setString("cgroup", parameterTool.get("cgroup"));
    conf.setString("confdir", parameterTool.get("confdir"));
    conf.setInteger("sourceParallelism", sourceParallelism);
    conf.setInteger("sinkParallelism", sinkParallelism);
    conf.setInteger("mapParallelism", mapParallelism);
    conf.setInteger("hdfsSinkParallelism", hdfsSinkParallelism);
    conf.setInteger("keybyms", keyByMilliseconds);

    env.getConfig().setGlobalJobParameters(conf);
    RocketMQSource rocketMQSource = new RocketMQSource();
    DataStream<String> rawSource = env
            .addSource(rocketMQSource).setParallelism(sourceParallelism).name("DAT_接收_Source");

    //原始Xml入Hdfs
    rawSource
            .addSink(HdfsSink.getSink(topic)).setParallelism(sinkParallelism).name("DAT_HDFS入库_Sink");

    try {
        env.execute("DAT Flow");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	final ParameterTool params = ParameterTool.fromArgs(args);
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	env.getConfig().setGlobalJobParameters(params); // make parameters available in the web interface

	double minSupport = params.getDouble("min-support", 0.5);
	int iterations = params.getInt("itemset-size", 4);

	if (!parametersCorrect(minSupport, iterations)) { return; }

	// load the data
	DataSet<Tuple2<Integer, Integer>> input = env.readCsvFile(params.getRequired("input"))
			.includeFields("11")
			.fieldDelimiter("\t")
			.lineDelimiter("\n")
			.types(Integer.class, Integer.class);

	// get the number of distinct transactions
	long numberOfTransactions = input
			.distinct(0)
			.count();
	// calculate the number of transactions sufficient for the support threshold
	long minNumberOfTransactions = (long) (numberOfTransactions * minSupport);

	DataSet<Tuple2<Integer, ArrayList<Integer>>> transactions = input
			.groupBy(0)
			.reduceGroup(new TransactionGroupReduceFunction());

	// compute frequent itemsets for itemset_size = 1
	DataSet<ItemSet> c1 = input
			// map item to 1
			.map(new InputMapFunction())
			// group by hashCode of the ItemSet
			.groupBy(new ItemSetKeySelector())
			// sum the number of transactions containing the ItemSet
			.reduce(new ItemSetReduceFunction())
			// remove ItemSets with frequency under the support threshold
			.filter(new ItemSetFrequencyFilterFunction(minNumberOfTransactions));

	// start of the loop
	// itemset_size = 2
	IterativeDataSet<ItemSet> initial = c1.iterate(iterations - 1);

	// create the candidate itemset for the next iteration
	DataSet<ItemSet> candidates = initial.cross(c1)
			.with(new ItemSetCrossFunction())
			.distinct(new ItemSetKeySelector());

	// calculate actual numberOfTransactions
	DataSet<ItemSet> selected = candidates
			.map(new ItemSetCalculateFrequency()).withBroadcastSet(transactions, "transactions")
			.filter(new ItemSetFrequencyFilterFunction(minNumberOfTransactions));

	// end of the loop
	// stop when we run out of iterations or candidates is empty
	DataSet<ItemSet> output = initial.closeWith(selected, selected);

	if (params.has("output")) {
		// write the final solution to file
		output.writeAsFormattedText(params.get("output"), new ItemSetTextFormatter());
		env.execute("Flink Apriori");
	} else {
		System.out.println("Printing result to stdout. Use --output to specify output path.");
		output.print();

		System.out.println("Number of iterations: " + iterations);
		System.out.println("Number of transactions: " + numberOfTransactions);

		System.out.println("Minimal number of transactions for support threshold of "
				+ minSupport + " = " + minNumberOfTransactions);
	}
}
 

import org.apache.flink.api.java.utils.ParameterTool; //导入方法依赖的package包/类
public ElasticsearchSinkBase(
	ElasticsearchApiCallBridge callBridge,
	Map<String, String> userConfig,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler) {

	this.callBridge = checkNotNull(callBridge);
	this.elasticsearchSinkFunction = checkNotNull(elasticsearchSinkFunction);
	this.failureHandler = checkNotNull(failureHandler);

	// we eagerly check if the user-provided sink function and failure handler is serializable;
	// otherwise, if they aren't serializable, users will merely get a non-informative error message
	// "ElasticsearchSinkBase is not serializable"

	checkArgument(InstantiationUtil.isSerializable(elasticsearchSinkFunction),
		"The implementation of the provided ElasticsearchSinkFunction is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	checkArgument(InstantiationUtil.isSerializable(failureHandler),
		"The implementation of the provided ActionRequestFailureHandler is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	// extract and remove bulk processor related configuration from the user-provided config,
	// so that the resulting user config only contains configuration related to the Elasticsearch client.

	checkNotNull(userConfig);

	ParameterTool params = ParameterTool.fromMap(userConfig);

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS)) {
		bulkProcessorFlushMaxActions = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
	} else {
		bulkProcessorFlushMaxActions = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB)) {
		bulkProcessorFlushMaxSizeMb = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
	} else {
		bulkProcessorFlushMaxSizeMb = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS)) {
		bulkProcessorFlushIntervalMillis = params.getInt(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
	} else {
		bulkProcessorFlushIntervalMillis = null;
	}

	boolean bulkProcessorFlushBackoffEnable = params.getBoolean(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, true);
	userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE);

	if (bulkProcessorFlushBackoffEnable) {
		this.bulkProcessorFlushBackoffPolicy = new BulkFlushBackoffPolicy();

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)) {
			bulkProcessorFlushBackoffPolicy.setBackoffType(FlushBackoffType.valueOf(params.get(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES)) {
			bulkProcessorFlushBackoffPolicy.setMaxRetryCount(params.getInt(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY)) {
			bulkProcessorFlushBackoffPolicy.setDelayMillis(params.getLong(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY);
		}

	} else {
		bulkProcessorFlushBackoffPolicy = null;
	}

	this.userConfig = userConfig;
}