Java ExecutionEnvironment类代码示例

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

	DataSet<Record> csvInput = env
			.readCsvFile("D://NOTBACKEDUP//dataflow//flink-table//src//main//resources//data//olympic-athletes.csv")
			.pojoType(Record.class, "playerName", "country", "year", "game", "gold", "silver", "bronze", "total");
	// register the DataSet athletes as table "athletes" with fields derived
	// from the dataset
	Table atheltes = tableEnv.fromDataSet(csvInput);
	tableEnv.registerTable("athletes", atheltes);
	// run a SQL query on the Table and retrieve the result as a new Table
	Table groupedByCountry = tableEnv.sql("SELECT country, SUM(total) as frequency FROM athletes group by country");

	DataSet<Result> result = tableEnv.toDataSet(groupedByCountry, Result.class);

	result.print();

	Table groupedByGame = atheltes.groupBy("game").select("game, total.sum as frequency");

	DataSet<GameResult> gameResult = tableEnv.toDataSet(groupedByGame, GameResult.class);

	gameResult.print();

}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
/**
 * This test validates that no exceptions is thrown when an empty dataset
 * calls maxBy().
 */
@Test
public void testMaxByKeyFieldsDataset() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.maxBy(4, 0, 1, 2, 3);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

        // parse parameters
        ParameterTool params = ParameterTool.fromArgs(args);
        // path to ratings.csv file
        String ratingsCsvPath = params.getRequired("input");

        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        DataSource<String> file = env.readTextFile(ratingsCsvPath);
        file.flatMap(new ExtractRating())
            .groupBy(0)
            // .reduceGroup(new SumRatingCount())
            .sum(1)
            .print();
    }
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
public static void main(String... args) throws  Exception {
    File txtFile = new File("/tmp/test/file.txt");
    File csvFile = new File("/tmp/test/file.csv");
    File binFile = new File("/tmp/test/file.bin");

    writeToFile(txtFile, "txt");
    writeToFile(csvFile, "csv");
    writeToFile(binFile, "bin");

    final ExecutionEnvironment env =
            ExecutionEnvironment.getExecutionEnvironment();
    final TextInputFormat format = new TextInputFormat(new Path("/tmp/test"));

    GlobFilePathFilter filesFilter = new GlobFilePathFilter(
            Collections.singletonList("**"),
            Arrays.asList("**/file.bin")
    );
    System.out.println(Arrays.toString(GlobFilePathFilter.class.getDeclaredFields()));
    format.setFilesFilter(filesFilter);

    DataSet<String> result = env.readFile(format, "/tmp");
    result.writeAsText("/temp/out");
    env.execute("GlobFilePathFilter-Test");
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

		// set up the execution environment
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		// get input data
		DataSet<String> text = env.fromElements(
				"To be, or not to be,--that is the question:--",
				"Whether 'tis nobler in the mind to suffer",
				"The slings and arrows of outrageous fortune",
				"Or to take arms against a sea of troubles,"
				);

		DataSet<Tuple2<String, Integer>> counts =
				// split up the lines in pairs (2-tuples) containing: (word,1)
				text.flatMap(new LineSplitter())
				// group by the tuple field "0" and sum up tuple field "1"
				.groupBy(0)
				.sum(1);

		// execute and print result
		counts.print();

	}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
/**
 * Test groupReduceOnNeighbors() -NeighborsFunction-
 * with an edge having a srcId that does not exist in the vertex DataSet.
 */
@Test
public void testGroupReduceOnNeighborsInvalidEdgeSrcId() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(PARALLELISM);
	env.getConfig().disableSysoutLogging();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeInvalidTrgData(env), env);

	try {
		DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
				graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.ALL);

		verticesWithSumOfAllNeighborValues.output(new DiscardingOutputFormat<>());
		env.execute();
	} catch (Exception e) {
		// We expect the job to fail with an exception
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
public static DataSet<IntValue> getIntDataSet(ExecutionEnvironment env) {
	List<IntValue> data = new ArrayList<>();

	data.add(new IntValue(1));
	data.add(new IntValue(2));
	data.add(new IntValue(2));
	data.add(new IntValue(3));
	data.add(new IntValue(3));
	data.add(new IntValue(3));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(4));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));
	data.add(new IntValue(5));

	Collections.shuffle(data);

	return env.fromCollection(data);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testGroupingKeyForwardIfNotUsed() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());

	DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
	Table table = tableEnv.fromDataSet(input, "a, b, c");

	Table result = table
			.groupBy("b").select("a.sum");

	DataSet<Row> ds = tableEnv.toDataSet(result, Row.class);
	List<Row> results = ds.collect();
	String expected = "1\n" + "5\n" + "15\n" + "34\n" + "65\n" + "111\n";
	compareResultAsText(results, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testWithDoubleValueMapper() throws Exception {
	/*
	 * Test create() with edge dataset and a mapper that assigns a double constant as value
     */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Double, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongEdgeData(env),
		new AssignDoubleValueMapper(), env);

	DataSet<Vertex<Long, Double>> data = graph.getVertices();
	List<Vertex<Long, Double>> result = data.collect();

	expectedResult = "1,0.1\n" +
		"2,0.1\n" +
		"3,0.1\n" +
		"4,0.1\n" +
		"5,0.1\n";

	compareResultAsTuples(result, expectedResult);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testWithDifferentType() throws Exception {
	/*
	 * Test joinWithVertices with the input DataSet passed as a parameter containing
	 * less elements than the vertex DataSet and of a different type(Boolean)
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithVertices(graph.getVertices().first(3)
		.map(new ProjectIdWithTrue()), new DoubleIfTrueMapper());

	DataSet<Vertex<Long, Long>> data = res.getVertices();
	List<Vertex<Long, Long>> result = data.collect();

	expectedResult = "1,2\n" +
		"2,4\n" +
		"3,6\n" +
		"4,4\n" +
		"5,5\n";

	compareResultAsTuples(result, expectedResult);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testIdentityIteration() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(43);
		
		IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
		iteration.closeWith(iteration).output(new DiscardingOutputFormat<Long>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testSumOfOutNeighborsNoValue() throws Exception {
	/*
	 * Get the sum of out-neighbor values
	 * for each vertex
        */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Tuple2<Long, Long>> verticesWithSumOfOutNeighborValues =
		graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.OUT);
	List<Tuple2<Long, Long>> result = verticesWithSumOfOutNeighborValues.collect();

	expectedResult = "1,5\n" +
		"2,3\n" +
		"3,9\n" +
		"4,5\n" +
		"5,1\n";

	compareResultAsTuples(result, expectedResult);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testOnTargetWithCustom() throws Exception {
	/*
     * Test joinWithEdgesOnTarget with a DataSet containing custom parametrised type input values
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdgesOnTarget(TestGraphUtils.getLongCustomTuple2TargetData(env),
		new CustomValueMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,10\n" +
		"1,3,20\n" +
		"2,3,20\n" +
		"3,4,40\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@SuppressWarnings("serial")
@Test
public void testFilterEdges() throws Exception {
	/*
	 * Test filterOnEdges:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.filterOnEdges(new FilterFunction<Edge<Long, Long>>() {
		public boolean filter(Edge<Long, Long> edge) throws Exception {
			return (edge.getValue() > 34);
		}
	}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入依赖的package包/类
@Test
public void testFromCollectionEdgesWithInitialValue() throws Exception {
	/*
        * Test fromCollection(edges) with vertices initialised by a
        * function that takes the id and doubles it
        */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongEdges(),
		new InitVerticesMapper(), env);

	DataSet<Vertex<Long, Long>> data = graph.getVertices();
	List<Vertex<Long, Long>> result = data.collect();

	expectedResult = "1,2\n" +
		"2,4\n" +
		"3,6\n" +
		"4,8\n" +
		"5,10\n";

	compareResultAsTuples(result, expectedResult);
}