Java DataSet.writeAsText方法代码示例

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testSerializeWithAvro() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map(new MapFunction<User, User>() {
				@Override
				public User map(User value) throws Exception {
					Map<CharSequence, Long> ab = new HashMap<CharSequence, Long>(1);
					ab.put("hehe", 12L);
					value.setTypeMap(ab);
					return value;
				}
			});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n" +
				"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n";

}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(String... args) throws  Exception {
    File txtFile = new File("/tmp/test/file.txt");
    File csvFile = new File("/tmp/test/file.csv");
    File binFile = new File("/tmp/test/file.bin");

    writeToFile(txtFile, "txt");
    writeToFile(csvFile, "csv");
    writeToFile(binFile, "bin");

    final ExecutionEnvironment env =
            ExecutionEnvironment.getExecutionEnvironment();
    final TextInputFormat format = new TextInputFormat(new Path("/tmp/test"));

    GlobFilePathFilter filesFilter = new GlobFilePathFilter(
            Collections.singletonList("**"),
            Arrays.asList("**/file.bin")
    );
    System.out.println(Arrays.toString(GlobFilePathFilter.class.getDeclaredFields()));
    format.setFilesFilter(filesFilter);

    DataSet<String> result = env.readFile(format, "/tmp");
    result.writeAsText("/temp/out");
    env.execute("GlobFilePathFilter-Test");
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testSimpleAvroRead() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			// null map type because the order changes in different JVMs (hard to test)
	.map(new MapFunction<User, User>() {
		@Override
		public User map(User value) throws Exception {
			value.setTypeMap(null);
			return value;
		}
	});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n" +
				"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": null, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n";
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testKeySelection() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS.groupBy("name").reduceGroup(new GroupReduceFunction<User, Tuple2<String, Integer>>() {
		@Override
		public void reduce(Iterable<User> values, Collector<Tuple2<String, Integer>> out) throws Exception {
			for (User u : values) {
				out.collect(new Tuple2<String, Integer>(u.getName().toString(), 1));
			}
		}
	});
	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Alyssa,1)\n(Charlie,1)\n";
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testStandardCountingWithCombiner() throws Exception{
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
			map(new Mapper1());

	DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
			groupBy(0).
			reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
					new SumReducer(), new SumReducer()));

	String resultPath = tempFolder.newFile().toURI().toString();

	counts.writeAsText(resultPath);
	env.execute();

	String expected = "(0,5)\n"+
			"(1,6)\n" +
			"(2,6)\n" +
			"(3,4)\n";

	compareResultsByLinesInMemory(expected, resultPath);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
/**
 * 
 * <pre>
 *     (SINK 3) (SINK 1)   (SINK 2) (SINK 4)
 *         \     /             \     /
 *         (SRC A)             (SRC B)
 * </pre>
 * 
 * NOTE: this case is currently not caught by the compiler. we should enable the test once it is caught.
 */
@Test
public void testBranchingDisjointPlan() {
	// construct the plan
	final String out1Path = "file:///test/1";
	final String out2Path = "file:///test/2";
	final String out3Path = "file:///test/3";
	final String out4Path = "file:///test/4";

	// construct the plan
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);
	DataSet<Long> sourceA = env.generateSequence(0,1);
	DataSet<Long> sourceB = env.generateSequence(0,1);

	sourceA.writeAsText(out1Path);
	sourceB.writeAsText(out2Path);
	sourceA.writeAsText(out3Path);
	sourceB.writeAsText(out4Path);

	Plan plan = env.createProgramPlan();
	compileNoStats(plan);

}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testUngroupedHadoopReducer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
			map(new Mapper2());

	DataSet<Tuple2<IntWritable, IntWritable>> sum = ds.
			reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
					new SumReducer(), new SumReducer()));

	String resultPath = tempFolder.newFile().toURI().toString();

	sum.writeAsText(resultPath);
	env.execute();

	String expected = "(0,231)\n";

	compareResultsByLinesInMemory(expected, resultPath);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testCombiner() throws Exception {
	org.junit.Assume.assumeThat(mode, new IsEqual<TestExecutionMode>(TestExecutionMode.CLUSTER));
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<IntWritable, IntWritable>> ds = HadoopTestData.getKVPairDataSet(env).
			map(new Mapper3());

	DataSet<Tuple2<IntWritable, IntWritable>> counts = ds.
			groupBy(0).
			reduceGroup(new HadoopReduceCombineFunction<IntWritable, IntWritable, IntWritable, IntWritable>(
					new SumReducer(), new KeyChangingReducer()));

	String resultPath = tempFolder.newFile().toURI().toString();

	counts.writeAsText(resultPath);
	env.execute();

	String expected = "(0,5)\n"+
			"(1,6)\n" +
			"(2,5)\n" +
			"(3,5)\n";

	compareResultsByLinesInMemory(expected, resultPath);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
private void testField(final String fieldName) throws Exception {
	before();

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Object> res = usersDS.groupBy(fieldName).reduceGroup(new GroupReduceFunction<User, Object>() {
		@Override
		public void reduce(Iterable<User> values, Collector<Object> out) throws Exception {
			for (User u : values) {
				out.collect(u.get(fieldName));
			}
		}
	});
	res.writeAsText(resultPath);
	env.execute("Simple Avro read job");

	// test if automatic registration of the Types worked
	ExecutionConfig ec = env.getConfig();
	Assert.assertTrue(ec.getRegisteredKryoTypes().contains(Fixed16.class));

	if (fieldName.equals("name")) {
		expected = "Alyssa\nCharlie";
	} else if (fieldName.equals("type_enum")) {
		expected = "GREEN\nRED\n";
	} else if (fieldName.equals("type_double_test")) {
		expected = "123.45\n1.337\n";
	} else {
		Assert.fail("Unknown field");
	}

	after();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testAggregatorWithParameterForIterateDelta() throws Exception {
	/*
	 * Test aggregator with parameter for iterateDelta
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);

	DataSet<Tuple2<Integer, Integer>> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env).map(new TupleMakerMap());

	DeltaIteration<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> iteration = initialSolutionSet.iterateDelta(
			initialSolutionSet, MAX_ITERATIONS, 0);

	// register aggregator
	LongSumAggregator aggr = new LongSumAggregatorWithParameter(4);
	iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);

	DataSet<Tuple2<Integer, Integer>> updatedDs = iteration.getWorkset().map(new AggregateMapDelta());

	DataSet<Tuple2<Integer, Integer>> newElements = updatedDs.join(iteration.getSolutionSet())
			.where(0).equalTo(0).flatMap(new UpdateFilter());

	DataSet<Tuple2<Integer, Integer>> iterationRes = iteration.closeWith(newElements, newElements);
	DataSet<Integer> result = iterationRes.map(new ProjectSecondMapper());
	result.writeAsText(resultPath);

	env.execute();

	expected = "1\n" + "2\n" + "2\n" + "3\n" + "3\n"
			+ "3\n" + "4\n" + "4\n" + "4\n" + "4\n"
			+ "5\n" + "5\n" + "5\n" + "5\n" + "5\n";
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Override
protected void testProgram() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> text = env.readTextFile(textPath);

	DataSet<WCBase> counts = text
			.flatMap(new Tokenizer())
			.groupBy("word")
			.reduce(new ReduceFunction<WCBase>() {
				private static final long serialVersionUID = 1L;
				public WCBase reduce(WCBase value1, WCBase value2) {
					WC wc1 = (WC) value1;
					WC wc2 = (WC) value2;
					int c = wc1.secretCount.getCount() + wc2.secretCount.getCount();
					wc1.secretCount.setCount(c);
					return wc1;
				}
			})
			.map(new MapFunction<WCBase, WCBase>() {
				@Override
				public WCBase map(WCBase value) throws Exception {
					WC wc = (WC) value;
					wc.count = wc.secretCount.getCount();
					return wc;
				}
			});

	counts.writeAsText(resultPath);

	env.execute("WordCount with custom data types example");
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Override
protected void testProgram() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> stringDs = env.fromElements("aa", "ab", "ac", "ad");
	DataSet<String> flatMappedDs = stringDs.flatMap((s, out) -> out.collect(s.replace("a", "b")));
	flatMappedDs.writeAsText(resultPath);
	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testSerializeWithAvro() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			// null map type because the order changes in different JVMs (hard to test)
			.map(new MapFunction<User, User>() {
				@Override
				public User map(User value) throws Exception {
					Map<CharSequence, Long> ab = new HashMap<CharSequence, Long>(1);
					ab.put("hehe", 12L);
					value.setTypeMap(ab);
					return value;
				}
			});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");


	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, \"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, \"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n" +
				"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, \"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}}\n";

}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testNonPassingMapper() throws Exception{
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
	DataSet<Tuple2<IntWritable, Text>> nonPassingFlatMapDs = ds.
			flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new NonPassingMapper()));

	String resultPath = tempFolder.newFile().toURI().toString();

	nonPassingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
	env.execute();

	compareResultsByLinesInMemory("\n", resultPath);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testWithAvroGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS.groupBy(new KeySelector<User, String>() {
		@Override
		public String getKey(User value) throws Exception {
			return String.valueOf(value.getName());
		}
	}).reduceGroup(new GroupReduceFunction<User, Tuple2<String, Integer>>() {
		@Override
		public void reduce(Iterable<User> values, Collector<Tuple2<String, Integer>> out) throws Exception {
			for(User u : values) {
				out.collect(new Tuple2<String, Integer>(u.getName().toString(), 1));
			}
		}
	});

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");


	expected = "(Charlie,1)\n(Alyssa,1)\n";
}