Java StreamExecutionEnvironment.socketTextStream方法代码示例

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		if (args.length != 2){
			System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
			return;
		}

		String hostName = args[0];
		Integer port = Integer.parseInt(args[1]);

		// set up the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment
				.getExecutionEnvironment();

		// get input data
		DataStream<String> text = env.socketTextStream(hostName, port);

		DataStream<Tuple2<String, Integer>> counts =
		// split up the lines in pairs (2-tuples) containing: (word,1)
		text.flatMap(new LineSplitter())
		// group by the tuple field "0" and sum up tuple field "1"
				.keyBy(0)
				.sum(1);

		counts.print();

		// execute program
		env.execute("Java WordCount from SocketTextStream Example");
	}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
					"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
					"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
					"type the input text into the command line");
			return;
		}

		DataServiceFacade dataService = new DataServiceFacade(DataEntityType.WORD_COUNT);

		dataService.setUpEmbeddedCassandra();
		dataService.setUpDataModel();

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordCount> result = text

				.flatMap(new FlatMapFunction<String, WordCount>() {
					@Override
					public void flatMap(String value, Collector<WordCount> out) {
						// normalize and split the line
						String[] words = value.toLowerCase().split("\\s");

						// emit the pairs
						for (String word : words) {
							if (!word.isEmpty()) {
								//Do not accept empty word, since word is defined as primary key in C* table
								out.collect(new WordCount(word, 1L));
							}
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordCount>() {
					@Override
					public WordCount reduce(WordCount a, WordCount b) {
						return new WordCount(a.getWord(), a.getCount() + b.getCount());
					}
				});

		CassandraSink.addSink(result)
				.setHost("127.0.0.1")
				.build();

		CQLPrintSinkFunction<WordCount, WordCount> func = new CQLPrintSinkFunction();
		func.setDataModel(dataService, 10);
		result.addSink(func).setParallelism(1);

		env.execute("Socket Window WordCount (POJO) w/ C* Sink");
	}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
					"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
					"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
					"type the input text into the command line");
			return;
		}

		DataServiceFacade dataService = new DataServiceFacade(DataEntityType.WORD_COUNT);

		dataService.setUpEmbeddedCassandra();
		dataService.setUpDataModel();

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<Tuple2<String, Long>> result = text

				.flatMap(new FlatMapFunction<String, Tuple2<String, Long>>() {
					@Override
					public void flatMap(String value, Collector<Tuple2<String, Long>> out) {
						// normalize and split the line
						String[] words = value.toLowerCase().split("\\s");

						// emit the pairs
						for (String word : words) {
							//Do not accept empty word, since word is defined as primary key in C* table
							if (!word.isEmpty()) {
								out.collect(new Tuple2<String, Long>(word, 1L));
							}
						}
					}
				})

				.keyBy(0)
				.timeWindow(Time.seconds(5))
				.sum(1)
				;

		CassandraSink.addSink(result)
				.setQuery("INSERT INTO " + WordCount.CQL_KEYSPACE_NAME + "." + WordCount.CQL_TABLE_NAME + "(word, count) " +
						"values (?, ?);")
				.setHost("127.0.0.1")
				.build();

		CQLPrintSinkFunction<Tuple2<String, Long>, WordCount> func = new CQLPrintSinkFunction();
		func.setDataModel(dataService, 10);
		result.addSink(func).setParallelism(1);

		env.execute("Socket Window WordCount (Tuple) w/ C* Sink");
	}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * The program main method.
 * @param args the command line arguments.
 */
public static void main(String[] args) throws Exception {

  // CONFIGURATION
  ParameterTool parameter = ParameterTool.fromArgs(args);
  final int port = Integer.valueOf(parameter.getRequired("port"));
  final Path outputPath = FileSystems.getDefault().getPath(parameter.get("output", PROGRAM_NAME + ".out"));
  final long windowSize = parameter.getLong("windowSize", 10);
  final TimeUnit windowUnit = TimeUnit.valueOf(parameter.get("windowUnit", "SECONDS"));
  final int parallelism = parameter.getInt("parallelism", 1);

  // ENVIRONMENT
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

  // CONFIGURATION RESUME
  System.out.println("############################################################################");
  System.out.printf("%s\n", PROGRAM_NAME);
  System.out.println("----------------------------------------------------------------------------");
  System.out.printf("%s\n", PROGRAM_DESCRIPTION);
  System.out.println("****************************************************************************");
  System.out.println("Port: " + port);
  System.out.println("Output: " + outputPath);
  System.out.println("Window: " + windowSize + " " + windowUnit);
  System.out.println("Parallelism: " + parallelism);
  System.out.println("############################################################################");

  // TOPOLOGY
  DataStream<String> text = env.socketTextStream("localhost", port, "\n");

  DataStream<WordWithCount> windowCounts = text
      .flatMap(new WordTokenizer())
      .keyBy("word")
      .timeWindow(Time.of(windowSize, windowUnit))
      .reduce(new WordCountReducer())
      .setParallelism(parallelism);

  windowCounts.writeAsText(outputPath.toAbsolutePath().toString(), FileSystem.WriteMode.OVERWRITE);

  // EXECUTION
  env.execute(PROGRAM_NAME);
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
				"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
				"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
				"type the input text into the command line");
			return;
		}

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordWithCount> windowCounts = text

				.flatMap(new FlatMapFunction<String, WordWithCount>() {
					@Override
					public void flatMap(String value, Collector<WordWithCount> out) {
						for (String word : value.split("\\s")) {
							out.collect(new WordWithCount(word, 1L));
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordWithCount>() {
					@Override
					public WordWithCount reduce(WordWithCount a, WordWithCount b) {
						return new WordWithCount(a.word, a.count + b.count);
					}
				});

		// print the results with a single thread, rather than in parallel
		windowCounts.print().setParallelism(1);

		env.execute("Socket Window WordCount");
	}