Java JobGraph.getVerticesSortedTopologicallyFromSources方法代码示例

import org.apache.flink.runtime.jobgraph.JobGraph; //导入方法依赖的package包/类
@Test
public void testInheritOverride() {
	// verify that we can explicitly disable inheritance of the input slot sharing groups

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should not inherit group but be in "default"
	src1.union(src2).filter(dummyFilter).slotSharingGroup("default");
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
}
 

import org.apache.flink.runtime.jobgraph.JobGraph; //导入方法依赖的package包/类
/**
 * Tests that there are no collisions with two identical sources.
 *
 * <pre>
 * [ (src0) ] --\
 *               +--> [ (sink) ]
 * [ (src1) ] --/
 * </pre>
 */
@Test
public void testNodeHashIdenticalSources() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src0 = env.addSource(new NoOpSourceFunction());
	DataStream<String> src1 = env.addSource(new NoOpSourceFunction());

	src0.union(src1).addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	assertTrue(vertices.get(0).isInputVertex());
	assertTrue(vertices.get(1).isInputVertex());

	assertNotNull(vertices.get(0).getID());
	assertNotNull(vertices.get(1).getID());

	assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID());
}
 

import org.apache.flink.runtime.jobgraph.JobGraph; //导入方法依赖的package包/类
@Test
public void testUnion() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10);
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");

	// this should not inherit group "src-1"
	src1.union(src2).filter(dummyFilter);

	DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should inherit "group-1" now
	src3.union(src4).filter(dummyFilter);

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	// first pipeline
	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());

	// second pipeline
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}
 

import org.apache.flink.runtime.jobgraph.JobGraph; //导入方法依赖的package包/类
/**
 * Verifies that the chain start/end is correctly set.
 */
@Test
public void testChainStartEndSetting() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	// fromElements -> CHAIN(Map -> Print)
	env.fromElements(1, 2, 3)
		.map(new MapFunction<Integer, Integer>() {
			@Override
			public Integer map(Integer value) throws Exception {
				return value;
			}
		})
		.print();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());

	List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
	JobVertex sourceVertex = verticesSorted.get(0);
	JobVertex mapPrintVertex = verticesSorted.get(1);

	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
	assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());

	StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
	StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
	Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
	StreamConfig printConfig = chainedConfigs.values().iterator().next();

	assertTrue(sourceConfig.isChainStart());
	assertTrue(sourceConfig.isChainEnd());

	assertTrue(mapConfig.isChainStart());
	assertFalse(mapConfig.isChainEnd());

	assertFalse(printConfig.isChainStart());
	assertTrue(printConfig.isChainEnd());
}
 

import org.apache.flink.runtime.jobgraph.JobGraph; //导入方法依赖的package包/类
@Test
public void testTwoPipelines() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	env.generateSequence(1, 10)
		.filter(dummyFilter).slotSharingGroup("isolated")
		.filter(dummyFilter).slotSharingGroup("default").disableChaining()
		.filter(dummyFilter).slotSharingGroup("group 1")
		.filter(dummyFilter).startNewChain()
		.print().disableChaining();

	// verify that a second pipeline does not inherit the groups from the first pipeline
	env.generateSequence(1, 10)
			.filter(dummyFilter).slotSharingGroup("isolated-2")
			.filter(dummyFilter).slotSharingGroup("default").disableChaining()
			.filter(dummyFilter).slotSharingGroup("group 2")
			.filter(dummyFilter).startNewChain()
			.print().disableChaining();

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
	assertNotEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertEquals(vertices.get(4).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(5).getSlotSharingGroup(), vertices.get(6).getSlotSharingGroup());

	int pipelineStart = 6;
	assertEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(pipelineStart + 2).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(pipelineStart + 1).getSlotSharingGroup());
	assertNotEquals(vertices.get(pipelineStart + 2).getSlotSharingGroup(), vertices.get(pipelineStart + 3).getSlotSharingGroup());
	assertEquals(vertices.get(pipelineStart + 3).getSlotSharingGroup(), vertices.get(pipelineStart + 4).getSlotSharingGroup());
	assertEquals(vertices.get(pipelineStart + 4).getSlotSharingGroup(), vertices.get(pipelineStart + 5).getSlotSharingGroup());

}