Java InputFormatVertex类代码示例

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static InputFormatVertex createEdgesInput(JobGraph jobGraph, String edgesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// edges
	@SuppressWarnings("unchecked")
	CsvInputFormat edgesInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class);
	InputFormatVertex edgesInput = JobGraphUtils.createInput(edgesInFormat, edgesPath, "EdgesInput", jobGraph,
		numSubTasks);
	TaskConfig edgesInputConfig = new TaskConfig(edgesInput.getConfiguration());
	{
		edgesInputConfig.setOutputSerializer(serializer);
		edgesInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		edgesInputConfig.setOutputComparator(comparator, 0);
	}

	return edgesInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		
		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
	}

	return pointsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static InputFormatVertex createCentersInput(JobGraph jobGraph, String centersPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);

		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
	}

	return modelsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSourceTask.class);
	vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	vertex.setInvokableClass(DataSourceTask.class);

	// set user code
	config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
	config.setStubParameters(node.getPactContract().getParameters());

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static <T extends InputFormat<?,?>> InputFormatVertex createInput(UserCodeWrapper<T> stub, String name, JobGraph graph,
		int degreeOfParallelism)
{
	InputFormatVertex inputVertex = new InputFormatVertex(name);
	graph.addVertex(inputVertex);
	
	inputVertex.setInvokableClass(DataSourceTask.class);
	inputVertex.setParallelism(degreeOfParallelism);

	TaskConfig inputConfig = new TaskConfig(inputVertex.getConfiguration());
	inputConfig.setStubWrapper(stub);
	
	return inputVertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static InputFormatVertex createVerticesInput(JobGraph jobGraph, String verticesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	@SuppressWarnings("unchecked")
	CsvInputFormat verticesInFormat = new CsvInputFormat(' ', LongValue.class);
	InputFormatVertex verticesInput = JobGraphUtils.createInput(verticesInFormat, verticesPath, "VerticesInput",
		jobGraph, numSubTasks);
	TaskConfig verticesInputConfig = new TaskConfig(verticesInput.getConfiguration());
	{
		verticesInputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		verticesInputConfig.setOutputSerializer(serializer);

		// chained mapper that duplicates the id
		TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
		chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IdDuplicator>(IdDuplicator.class));
		chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		chainedMapperConfig.setInputSerializer(serializer, 0);

		chainedMapperConfig.setOutputSerializer(serializer);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.setOutputComparator(comparator, 0);
		chainedMapperConfig.setOutputComparator(comparator, 1);

		verticesInputConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "ID Duplicator");
	}

	return verticesInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat pointsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "Input[Points]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
	}

	return pointsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static InputFormatVertex createModelsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat modelsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, pointsPath, "Input[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
		taskConfig.setOutputSerializer(serializer);
	}

	return modelsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private JobGraph createJobGraphV1(String pointsPath, String centersPath, String resultPath, int numSubTasks) {

		// -- init -------------------------------------------------------------------------------------------------
		final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();

		JobGraph jobGraph = new JobGraph("Distance Builder");

		// -- vertices ---------------------------------------------------------------------------------------------
		InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
		InputFormatVertex models = createModelsInput(jobGraph, centersPath, numSubTasks, serializer);
		AbstractJobVertex mapper = createMapper(jobGraph, numSubTasks, serializer);
		OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);

		// -- edges ------------------------------------------------------------------------------------------------
		JobGraphUtils.connect(points, mapper, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		JobGraphUtils.connect(models, mapper, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		JobGraphUtils.connect(mapper, output, ChannelType.NETWORK, DistributionPattern.POINTWISE);

		// -- instance sharing -------------------------------------------------------------------------------------
		
		SlotSharingGroup sharing = new SlotSharingGroup();
		
		points.setSlotSharingGroup(sharing);
		models.setSlotSharingGroup(sharing);
		mapper.setSlotSharingGroup(sharing);
		output.setSlotSharingGroup(sharing);

		return jobGraph;
	}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
public JobGraph createJobGraphUnifiedTails(
		String verticesPath, String edgesPath, String resultPath, int numSubTasks, int maxIterations)
{
	// -- init -------------------------------------------------------------------------------------------------
	final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
	@SuppressWarnings("unchecked")
	final TypeComparatorFactory<?> comparator =
		new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
	final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();

	JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");

	// -- invariant vertices -----------------------------------------------------------------------------------
	InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
	InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
	AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);

	AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
	TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());

	OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
	AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);

	// --------------- the tail (solution set join) ---------------
	AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph, numSubTasks);
	TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
	{
		tailConfig.setIterationId(ITERATION_ID);

		tailConfig.setIsWorksetIteration();
		tailConfig.setIsWorksetUpdate();

		tailConfig.setIsSolutionSetUpdate();
		tailConfig.setIsSolutionSetUpdateWithoutReprobe();

		// inputs and driver
		tailConfig.addInputToGroup(0);
		tailConfig.setInputSerializer(serializer, 0);

		// output
		tailConfig.setOutputSerializer(serializer);

		// the driver
		tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
		tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
		tailConfig.setDriverComparator(comparator, 0);
		tailConfig.setDriverPairComparator(pairComparator);
		
		tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
	}

	// -- edges ------------------------------------------------------------------------------------------------
	JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

	JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);

	JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
	tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);

	JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

	JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	vertices.setSlotSharingGroup(sharingGroup);
	edges.setSlotSharingGroup(sharingGroup);
	head.setSlotSharingGroup(sharingGroup);
	intermediate.setSlotSharingGroup(sharingGroup);
	tail.setSlotSharingGroup(sharingGroup);
	output.setSlotSharingGroup(sharingGroup);
	sync.setSlotSharingGroup(sharingGroup);
	
	intermediate.setStrictlyCoLocatedWith(head);
	tail.setStrictlyCoLocatedWith(head);

	return jobGraph;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入依赖的package包/类
private static JobGraph createJobGraph(String pointsPath, String centersPath, String resultPath, int numSubTasks, int numIterations) {

		// -- init -------------------------------------------------------------------------------------------------
		final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
		@SuppressWarnings("unchecked")
		final TypeComparatorFactory<?> int0Comparator = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class });

		JobGraph jobGraph = new JobGraph("KMeans Iterative");

		// -- vertices ---------------------------------------------------------------------------------------------
		InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
		InputFormatVertex centers = createCentersInput(jobGraph, centersPath, numSubTasks, serializer);
		
		AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer);
		AbstractJobVertex mapper = createMapper(jobGraph, numSubTasks, serializer, serializer, serializer, int0Comparator);
		
		AbstractJobVertex reducer = createReducer(jobGraph, numSubTasks, serializer, int0Comparator, serializer);
		
		AbstractJobVertex sync = createSync(jobGraph, numIterations, numSubTasks);
		
		OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);

		// -- edges ------------------------------------------------------------------------------------------------
		JobGraphUtils.connect(points, mapper, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(centers, head, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(head, mapper, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		new TaskConfig(mapper.getConfiguration()).setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);
		new TaskConfig(mapper.getConfiguration()).setInputCached(0, true);
		new TaskConfig(mapper.getConfiguration()).setRelativeInputMaterializationMemory(0,
				MEMORY_FRACTION_PER_CONSUMER);

		JobGraphUtils.connect(mapper, reducer, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		new TaskConfig(reducer.getConfiguration()).setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);
		
		JobGraphUtils.connect(head, output, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

		// -- instance sharing -------------------------------------------------------------------------------------
		
		SlotSharingGroup sharingGroup = new SlotSharingGroup();
		
		points.setSlotSharingGroup(sharingGroup);
		centers.setSlotSharingGroup(sharingGroup);
		head.setSlotSharingGroup(sharingGroup);
		mapper.setSlotSharingGroup(sharingGroup);
		reducer.setSlotSharingGroup(sharingGroup);
		sync.setSlotSharingGroup(sharingGroup);
		output.setSlotSharingGroup(sharingGroup);
		
		mapper.setStrictlyCoLocatedWith(head);
		reducer.setStrictlyCoLocatedWith(head);

		return jobGraph;
	}