Java ExecutionEnvironment.getExecutionEnvironment方法代码示例

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

	DataSet<Record> csvInput = env
			.readCsvFile("D://NOTBACKEDUP//dataflow//flink-table//src//main//resources//data//olympic-athletes.csv")
			.pojoType(Record.class, "playerName", "country", "year", "game", "gold", "silver", "bronze", "total");
	// register the DataSet athletes as table "athletes" with fields derived
	// from the dataset
	Table atheltes = tableEnv.fromDataSet(csvInput);
	tableEnv.registerTable("athletes", atheltes);
	// run a SQL query on the Table and retrieve the result as a new Table
	Table groupedByCountry = tableEnv.sql("SELECT country, SUM(total) as frequency FROM athletes group by country");

	DataSet<Result> result = tableEnv.toDataSet(groupedByCountry, Result.class);

	result.print();

	Table groupedByGame = atheltes.groupBy("game").select("game, total.sum as frequency");

	DataSet<GameResult> gameResult = tableEnv.toDataSet(groupedByGame, GameResult.class);

	gameResult.print();

}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

        // parse parameters
        ParameterTool params = ParameterTool.fromArgs(args);
        // path to ratings.csv file
        String ratingsCsvPath = params.getRequired("input");

        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSource<String> file = env.readTextFile(ratingsCsvPath);
        file.flatMap(new ExtractRating())
            .groupBy(0)
            // .reduceGroup(new SumRatingCount())
            .sum(1)
            .print();
    }
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String... args) throws  Exception {
    File txtFile = new File("/tmp/test/file.txt");
    File csvFile = new File("/tmp/test/file.csv");
    File binFile = new File("/tmp/test/file.bin");

    writeToFile(txtFile, "txt");
    writeToFile(csvFile, "csv");
    writeToFile(binFile, "bin");

    final ExecutionEnvironment env =
            ExecutionEnvironment.getExecutionEnvironment();
    final TextInputFormat format = new TextInputFormat(new Path("/tmp/test"));

    GlobFilePathFilter filesFilter = new GlobFilePathFilter(
            Collections.singletonList("**"),
            Arrays.asList("**/file.bin")
    );
    System.out.println(Arrays.toString(GlobFilePathFilter.class.getDeclaredFields()));
    format.setFilesFilter(filesFilter);

    DataSet<String> result = env.readFile(format, "/tmp");
    result.writeAsText("/temp/out");
    env.execute("GlobFilePathFilter-Test");
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

		// set up the execution environment
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		// get input data
		DataSet<String> text = env.fromElements(
				"To be, or not to be,--that is the question:--",
				"Whether 'tis nobler in the mind to suffer",
				"The slings and arrows of outrageous fortune",
				"Or to take arms against a sea of troubles,"
				);

		DataSet<Tuple2<String, Integer>> counts =
				// split up the lines in pairs (2-tuples) containing: (word,1)
				text.flatMap(new LineSplitter())
				// group by the tuple field "0" and sum up tuple field "1"
				.groupBy(0)
				.sum(1);

		// execute and print result
		counts.print();

	}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testTupleTwoOrderExp() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput("f1", Order.ASCENDING)
			.sortLocalOutput("f4", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testDisjunctivePreds() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());

	DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
	Table table = tableEnv.fromDataSet(input, "a, b, c");

	Table result = table
		.filter("a < 2 || a > 20");

	DataSet<Row> ds = tableEnv.toDataSet(result, Row.class);
	List<Row> results = ds.collect();
	String expected = "1,1,Hi\n" + "21,6,Comment#15\n";
	compareResultAsText(results, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testBatchTableSourceSQL() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
	BatchTableSource csvTable = CommonTestData.getCsvTableSource();

	tableEnv.registerTableSource("persons", csvTable);

	Table result = tableEnv
		.sql("SELECT `last`, FLOOR(id), score * 2 FROM persons WHERE score < 20");

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected = "Smith,1,24.6\n" +
		"Miller,3,15.78\n" +
		"Smith,4,0.24\n" +
		"Miller,6,13.56\n" +
		"Williams,8,4.68\n";

	compareResultAsText(results, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testJoinWithAtomicType2() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ds1 = env.fromElements(1, 2);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);

	DataSet<Tuple2<Integer, Tuple3<Integer, Long, String>>> joinDs = ds1
			.fullOuterJoin(ds2)
			.where("*")
			.equalTo(0)
			.with(new ProjectBothFunction<Integer, Tuple3<Integer, Long, String>>())
			.returns("Tuple2<java.lang.Object,java.lang.Object>");

	List<Tuple2<Integer, Tuple3<Integer, Long, String>>> result = joinDs.collect();

	String expected = "1,(1,1,Hi)\n" +
			"2,(2,2,Hello)\n" +
			"null,(3,2,Hello world)\n";

	compareResultAsTuples(result, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testNonPassingFlatMap() throws Exception {
	/*
	 * Test non-passing flatmap
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
	DataSet<String> nonPassingFlatMapDs = ds.
			flatMap(new FlatMapper1());

	List<String> result = nonPassingFlatMapDs.collect();

	String expected = "\n";

	compareResultAsText(result, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testIntegerBiggerThan128() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());

	DataSet<Tuple3<Integer, Long, String>> input = env.fromElements(new Tuple3<>(300, 1L, "Hello"));
	Table table = tableEnv.fromDataSet(input, "a, b, c");

	Table result = table
		.filter("a = 300 ");

	DataSet<Row> ds = tableEnv.toDataSet(result, Row.class);
	List<Row> results = ds.collect();
	String expected = "300,1,Hello\n";
	compareResultAsText(results, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testFilterWithoutExtended() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

	DataSet<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1,"d"));
	Table table = tableEnv
		.fromDataSet(input, "a, b")
		.filter("a % 2 = 0");

	String result = tableEnv.explain(table).replaceAll("\\r\\n", "\n");
	try (Scanner scanner = new Scanner(new File(testFilePath +
		"../../src/test/scala/resources/testFilter0.out"))){
		String source = scanner.useDelimiter("\\A").next().replaceAll("\\r\\n", "\n");
		assertEquals(source, result);
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test(expected = ValidationException.class)
public void testJoinWithNonMatchingKeyTypes() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);

	Table in1 = tableEnv.fromDataSet(ds1, "a, b, c");
	Table in2 = tableEnv.fromDataSet(ds2, "d, e, f, g, h");

	Table result = in1.join(in2)
		// Must fail. Types of join fields are not compatible (Integer and String)
		.where("a === g").select("c, g");

	tableEnv.toDataSet(result, Row.class).collect();
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Override
protected void testProgram() throws Exception {
	// set up execution environment
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// read vertex and edge data
	DataSet<Tuple1<Long>> vertices = env.readCsvFile(verticesPath).types(Long.class);

	DataSet<Tuple2<Long, Long>> edges = env.readCsvFile(edgesPath).fieldDelimiter(" ").types(Long.class, Long.class)
			.flatMap(new ConnectedComponents.UndirectEdge());

	// assign the initial components (equal to the vertex id)
	DataSet<Tuple2<Long, Long>> verticesWithInitialId = vertices.map(new DuplicateValue<Long>());

	// open a delta iteration
	DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
			verticesWithInitialId.iterateDelta(verticesWithInitialId, 100, 0);

	// apply the step logic: join with the edges, select the minimum neighbor, update if the component of the candidate is smaller
	DataSet<Tuple2<Long, Long>> changes = iteration.getWorkset().join(edges).where(0).equalTo(0).with(new ConnectedComponents.NeighborWithComponentIDJoin())
			.groupBy(0).aggregate(Aggregations.MIN, 1)
			.join(iteration.getSolutionSet()).where(0).equalTo(0)
			.with(new ConnectedComponents.ComponentIdFilter());

	// close the delta iteration (delta and new workset are identical)
	DataSet<Tuple2<Long, Long>> result = iteration.closeWith(changes, changes);

	result.writeAsCsv(resultPath, "\n", " ");

	// execute program
	env.execute("Connected Components Example");
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSource<String> lines = env.fromElements(
        "Apache Flink is a community-driven open source framework for distributed big data analytics,",
        "like Hadoop and Spark. The core of Apache Flink is a distributed streaming dataflow engine written",
        " in Java and Scala.[1][2] It aims to bridge the gap between MapReduce-like systems and shared-nothing",
        "parallel database systems. Therefore, Flink executes arbitrary dataflow programs in a data-parallel and",
        "pipelined manner.[3] Flink's pipelined runtime system enables the execution of bulk/batch and stream",
        "processing programs.[4][5] Furthermore, Flink's runtime supports the execution of iterative algorithms natively.[6]"
    );

    lines.flatMap((line, out) -> {
        String[] words = line.split("\\W+");
        for (String word : words) {
            out.collect(new Tuple2<>(word, 1));
        }
    })
    .returns(new TupleTypeInfo(TypeInformation.of(String.class), TypeInformation.of(Integer.class)))
    .groupBy(0)
    .sum(1)
    .print();
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String... args) throws Exception {

        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        List<Vertex<Integer, String>> vertices = Arrays.asList(
                new Vertex<>(1, "1"),
                new Vertex<>(2, "2"),
                new Vertex<>(3, "3"),
                new Vertex<>(4, "4"),
                new Vertex<>(5, "5")
        );

        List<Edge<Integer, Double>> edges = Arrays.asList(
                new Edge<>(1, 2, 2.0),
                new Edge<>(1, 3, 7.0),
                new Edge<>(2, 3, 2.0),
                new Edge<>(3, 2, 5.0),
                new Edge<>(2, 4, 4.0),
                new Edge<>(3, 4, 6.0),
                new Edge<>(3, 5, 3.0),
                new Edge<>(4, 5, 4.0),
                new Edge<>(5, 4, 1.0),
                new Edge<>(5, 1, 8.0)
        );

        Graph<Integer, String, Double> graph = Graph.fromCollection(vertices, edges, env);

        graph.run(new ShortestPath<>(1, 10)).print();
    }