Java ExecutionEnvironment.fromCollection方法代码示例

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);

	// should not work: sorted groups on groupings by key selectors
	longDs.groupBy(new KeySelector<Long, Long>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Long getKey(Long value) {
			return value;
		}

	}).sortGroup(0, Order.ASCENDING);

}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple2<byte[], Integer>> getTuple2WithByteArrayDataSet(ExecutionEnvironment env) {
	List<Tuple2<byte[], Integer>> data = new ArrayList<>();
	data.add(new Tuple2<>(new byte[]{0, 4}, 1));
	data.add(new Tuple2<>(new byte[]{2, 0}, 1));
	data.add(new Tuple2<>(new byte[]{2, 0, 4}, 4));
	data.add(new Tuple2<>(new byte[]{2, 1}, 3));
	data.add(new Tuple2<>(new byte[]{0}, 0));
	data.add(new Tuple2<>(new byte[]{2, 0}, 1));

	TupleTypeInfo<Tuple2<byte[], Integer>> type = new TupleTypeInfo<>(
			PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testAggregationTypes() {
	try {
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work: multiple aggregates
		tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4);

		// should work: nested aggregates
		tupleDs.aggregate(Aggregations.MIN, 2).aggregate(Aggregations.SUM, 1);

		// should not work: average on string
		try {
			tupleDs.aggregate(Aggregations.SUM, 2);
			Assert.fail();
		} catch (UnsupportedAggregationTypeException iae) {
			// we're good here
		}
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testMap() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());

	List<Tuple2<Integer, Map<String, String>>> rows = new ArrayList<>();
	rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
	rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));

	TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
		BasicTypeInfo.INT_TYPE_INFO,
		new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));

	DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
	tableEnv.registerDataSet("t1", ds1, "a, b");

	String sqlQuery = "SELECT b['foo'] FROM t1";
	Table result = tableEnv.sql(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "bar\n" + "spam\n";
	compareResultAsText(results, expected);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<PojoWithDateAndEnum> getPojoWithDateAndEnum(ExecutionEnvironment env) {
	List<PojoWithDateAndEnum> data = new ArrayList<PojoWithDateAndEnum>();

	PojoWithDateAndEnum one = new PojoWithDateAndEnum();
	one.group = new StringValue("a");
	one.date = new Date(666);
	one.cat = Category.CAT_A;
	data.add(one);

	PojoWithDateAndEnum two = new PojoWithDateAndEnum();
	two.group = new StringValue("a");
	two.date = new Date(666);
	two.cat = Category.CAT_A;
	data.add(two);

	PojoWithDateAndEnum three = new PojoWithDateAndEnum();
	three.group = new StringValue("b");
	three.date = new Date(666);
	three.cat = Category.CAT_B;
	data.add(three);

	return env.fromCollection(data);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple2<IntWritable, Text>> getKVPairDataSet(ExecutionEnvironment env) {

		List<Tuple2<IntWritable, Text>> data = new ArrayList<Tuple2<IntWritable, Text>>();
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(1), new Text("Hi")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(2), new Text("Hello")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(3), new Text("Hello world")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(4), new Text("Hello world, how are you?")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(5), new Text("I am fine.")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(6), new Text("Luke Skywalker")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(7), new Text("Comment#1")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(8), new Text("Comment#2")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(9), new Text("Comment#3")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(10), new Text("Comment#4")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(11), new Text("Comment#5")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(12), new Text("Comment#6")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(13), new Text("Comment#7")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(14), new Text("Comment#8")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(15), new Text("Comment#9")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(16), new Text("Comment#10")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(17), new Text("Comment#11")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(18), new Text("Comment#12")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(19), new Text("Comment#13")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(20), new Text("Comment#14")));
		data.add(new Tuple2<IntWritable, Text>(new IntWritable(21), new Text("Comment#15")));

		Collections.shuffle(data);

		return env.fromCollection(data);
	}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
private void testFullOuterStrategies(JoinHint hint) {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.fullOuterJoin(ds2, hint)
				.where(0).equalTo(4)
				.with(new DummyJoin());
	}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple2<Long, Long>> getDefaultEdgeDataSet(ExecutionEnvironment env) {

		List<Tuple2<Long, Long>> edges = new ArrayList<Tuple2<Long, Long>>();
		for (Object[] e : EDGES) {
			edges.add(new Tuple2<Long, Long>((Long) e[0], (Long) e[1]));
		}
		return env.fromCollection(edges);
	}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple3<IntValue, CrazyNested, POJO>> getTupleContainingPojos(ExecutionEnvironment env) {
	List<Tuple3<IntValue, CrazyNested, POJO>> data = new ArrayList<>();
	data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L))); // 3x
	data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
	data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
	// POJO is not initialized according to the first two fields.
	data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(2), new CrazyNested("two", "duo", 2L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L))); // 1x
	return env.fromCollection(data);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyAtomicInvalidExpression3() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 1);

	ds1.coGroup(ds2).where("myInt").equalTo("invalidKey");
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
/**
 * A graph that has at least one vertex with no ingoing/outgoing edges.
 */
public static DataSet<Edge<Long, Long>> getLongLongEdgeDataWithZeroDegree(
		ExecutionEnvironment env) {
	List<Edge<Long, Long>> edges = new ArrayList<>();
	edges.add(new Edge<>(1L, 2L, 12L));
	edges.add(new Edge<>(1L, 4L, 14L));
	edges.add(new Edge<>(1L, 5L, 15L));
	edges.add(new Edge<>(2L, 3L, 23L));
	edges.add(new Edge<>(3L, 5L, 35L));
	edges.add(new Edge<>(4L, 5L, 45L));

	return env.fromCollection(edges);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
private static DataSet<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> getSmallTuplebasedDataSetMatchingPojo(ExecutionEnvironment env) {
	List<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> data = new ArrayList<>();

	data.add(new Tuple7<>(new LongValue(10000L), new IntValue(10), new IntValue(100), new LongValue(1000L), new StringValue("One"), new IntValue(1), new StringValue("First")));
	data.add(new Tuple7<>(new LongValue(20000L), new IntValue(20), new IntValue(200), new LongValue(2000L), new StringValue("Two"), new IntValue(2), new StringValue("Second")));
	data.add(new Tuple7<>(new LongValue(30000L), new IntValue(30), new IntValue(300), new LongValue(3000L), new StringValue("Three"), new IntValue(3), new StringValue("Third")));

	return env.fromCollection(data);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple3<Long, Long, DummyCustomParameterizedType<Float>>> getLongLongCustomTuple3Data(
		ExecutionEnvironment env) {
	List<Tuple3<Long, Long, DummyCustomParameterizedType<Float>>> tuples = new ArrayList<>();
	tuples.add(new Tuple3<>(1L, 2L, new DummyCustomParameterizedType<>(10, 10f)));
	tuples.add(new Tuple3<>(1L, 3L, new DummyCustomParameterizedType<>(20, 20f)));
	tuples.add(new Tuple3<>(2L, 3L, new DummyCustomParameterizedType<>(30, 30f)));
	tuples.add(new Tuple3<>(3L, 4L, new DummyCustomParameterizedType<>(40, 40f)));

	return env.fromCollection(tuples);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
public static DataSet<Tuple7<Long, Integer, Integer, Long, String, Integer, String>> getSmallTuplebasedDataSetMatchingPojo(ExecutionEnvironment env) {
	List<Tuple7<Long, Integer, Integer, Long, String, Integer, String>> data = new ArrayList<>();
	data.add(new Tuple7<>(10000L, 10, 100, 1000L, "One", 1, "First"));
	data.add(new Tuple7<>(20000L, 20, 200, 2000L, "Two", 2, "Second"));
	data.add(new Tuple7<>(30000L, 30, 300, 3000L, "Three", 3, "Third"));

	return env.fromCollection(data);
}
 

import org.apache.flink.api.java.ExecutionEnvironment; //导入方法依赖的package包/类
/**
 * This test validates that an index which is out of bounds throws an
 * IndexOutOfBoundsException.
 */
@Test(expected = IndexOutOfBoundsException.class)
public void testOutOfTupleBoundsDataset3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, key out of tuple bounds
	tupleDs.minBy(1, 2, 3, 4, -1);
}