Java DataSet.filter方法代码示例

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testAllRejectingFilter() throws Exception {
	/*
	 * Test all-rejecting filter.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new Filter1());

	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "\n";

	compareResultAsTuples(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testFilterOnStringTupleField() throws Exception {
	/*
	 * Test filter on String tuple field.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new Filter3());
	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "3,2,Hello world\n"
			+
			"4,3,Hello world, how are you?\n";

	compareResultAsTuples(result, expected);

}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testFilterOnIntegerTupleField() throws Exception {
	/*
	 * Test filter on Integer tuple field.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new Filter4());
	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "2,2,Hello\n" +
			"4,3,Hello world, how are you?\n" +
			"6,3,Luke Skywalker\n" +
			"8,4,Comment#2\n" +
			"10,4,Comment#4\n" +
			"12,5,Comment#6\n" +
			"14,5,Comment#8\n" +
			"16,6,Comment#10\n" +
			"18,6,Comment#12\n" +
			"20,6,Comment#14\n";

	compareResultAsTuples(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testFilterBasicType() throws Exception {
	/*
	 * Test filter on basic type
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
	DataSet<String> filterDs = ds.
			filter(new Filter5());
	List<String> result = filterDs.collect();

	String expected = "Hi\n" +
			"Hello\n" +
			"Hello world\n" +
			"Hello world, how are you?\n";

	compareResultAsText(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testFilterOnCustomType() throws Exception {
	/*
	 * Test filter on custom type
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> filterDs = ds.
			filter(new Filter6());
	List<CustomType> result = filterDs.collect();

	String expected = "3,3,Hello world, how are you?\n"
			+
			"3,4,I am fine.\n" +
			"3,5,Luke Skywalker\n";

	compareResultAsText(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(final String[] args) throws Exception {

		final ParameterTool params = ParameterTool.fromArgs(args);

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		// get the data set
		final DataSet<StringTriple> file = getDataSet(env, params);

		// filter lines with empty fields
		final DataSet<StringTriple> filteredLines = file.filter(new EmptyFieldFilter());

		// Here, we could do further processing with the filtered lines...
		JobExecutionResult result;
		// output the filtered lines
		if (params.has("output")) {
			filteredLines.writeAsCsv(params.get("output"));
			// execute program
			result = env.execute("Accumulator example");
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			filteredLines.print();
			result = env.getLastJobExecutionResult();
		}

		// get the accumulator result via its registration key
		final List<Integer> emptyFields = result.getAccumulatorResult(EMPTY_FIELD_ACCUMULATOR);
		System.out.format("Number of detected empty fields per column: %s\n", emptyFields);
	}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testAllPassingFilter() throws Exception {
	/*
	 * Test all-passing filter.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new Filter2());
	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "1,1,Hi\n" +
			"2,2,Hello\n" +
			"3,2,Hello world\n" +
			"4,3,Hello world, how are you?\n" +
			"5,3,I am fine.\n" +
			"6,3,Luke Skywalker\n" +
			"7,4,Comment#1\n" +
			"8,4,Comment#2\n" +
			"9,4,Comment#3\n" +
			"10,4,Comment#4\n" +
			"11,5,Comment#5\n" +
			"12,5,Comment#6\n" +
			"13,5,Comment#7\n" +
			"14,5,Comment#8\n" +
			"15,5,Comment#9\n" +
			"16,6,Comment#10\n" +
			"17,6,Comment#11\n" +
			"18,6,Comment#12\n" +
			"19,6,Comment#13\n" +
			"20,6,Comment#14\n" +
			"21,6,Comment#15\n";

	compareResultAsTuples(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> input = env.fromElements("Please filter", "the words", "but not this");

	DataSet<String> output = input.filter(UtilFunction.getWordFilter());
	output.print();

	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> input = env.fromElements("Please filter", "the words", "but not this");

	FilterFunction<String> filter = (v) -> WordFilter.filter(v);

	DataSet<String> output = input.filter(filter);
	output.print();

	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> input = env.fromElements("Please filter", "the words", "but not this");

	DataSet<String> output = input.filter((v) -> WordFilter.filter(v));
	output.print();

	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> input = env.fromElements("Please filter", "the words", "but not this");

	DataSet<String> output = input.filter(UtilFunctionWrapper.UtilFunction.getWordFilter());
	output.print();

	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Override
protected void testProgram() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(value -> value.f2.contains("world"));
	filterDs.writeAsCsv(resultPath);
	env.execute();
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testBulkIterationWithTerminationCriterion() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();

		IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(100);

		DataSet<Integer> iterationResult = iteration.map(new AddSuperstepNumberMapper());

		DataSet<Integer> terminationCriterion = iterationResult.filter(new FilterFunction<Integer>() {
			public boolean filter(Integer value) {
				return value < 50;
			}
		});

		List<Integer> collected = new ArrayList<Integer>();

		iteration.closeWith(iterationResult, terminationCriterion)
				.output(new LocalCollectionOutputFormat<Integer>(collected));

		env.execute();

		assertEquals(1, collected.size());
		assertEquals(56, collected.get(0).intValue());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
/**
 * Apply filtering functions to the graph and return a sub-graph that
 * satisfies the predicates for both vertices and edges.
 *
 * @param vertexFilter the filter function for vertices.
 * @param edgeFilter the filter function for edges.
 * @return the resulting sub-graph.
 */
public Graph<K, VV, EV> subgraph(FilterFunction<Vertex<K, VV>> vertexFilter, FilterFunction<Edge<K, EV>> edgeFilter) {

	DataSet<Vertex<K, VV>> filteredVertices = this.vertices.filter(vertexFilter);

	DataSet<Edge<K, EV>> remainingEdges = this.edges.join(filteredVertices)
			.where(0).equalTo(0).with(new ProjectEdge<>())
			.join(filteredVertices).where(1).equalTo(0)
			.with(new ProjectEdge<>()).name("Subgraph");

	DataSet<Edge<K, EV>> filteredEdges = remainingEdges.filter(edgeFilter);

	return new Graph<>(filteredVertices, filteredEdges, this.context);
}