Java InputFormatVertex.setSlotSharingGroup方法代码示例

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private JobGraph createJobGraphV1(String pointsPath, String centersPath, String resultPath, int numSubTasks) {

		// -- init -------------------------------------------------------------------------------------------------
		final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();

		JobGraph jobGraph = new JobGraph("Distance Builder");

		// -- vertices ---------------------------------------------------------------------------------------------
		InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
		InputFormatVertex models = createModelsInput(jobGraph, centersPath, numSubTasks, serializer);
		AbstractJobVertex mapper = createMapper(jobGraph, numSubTasks, serializer);
		OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);

		// -- edges ------------------------------------------------------------------------------------------------
		JobGraphUtils.connect(points, mapper, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		JobGraphUtils.connect(models, mapper, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		JobGraphUtils.connect(mapper, output, ChannelType.NETWORK, DistributionPattern.POINTWISE);

		// -- instance sharing -------------------------------------------------------------------------------------
		
		SlotSharingGroup sharing = new SlotSharingGroup();
		
		points.setSlotSharingGroup(sharing);
		models.setSlotSharingGroup(sharing);
		mapper.setSlotSharingGroup(sharing);
		output.setSlotSharingGroup(sharing);

		return jobGraph;
	}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
public JobGraph createJobGraphUnifiedTails(
		String verticesPath, String edgesPath, String resultPath, int numSubTasks, int maxIterations)
{
	// -- init -------------------------------------------------------------------------------------------------
	final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
	@SuppressWarnings("unchecked")
	final TypeComparatorFactory<?> comparator =
		new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
	final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();

	JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");

	// -- invariant vertices -----------------------------------------------------------------------------------
	InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
	InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
	AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);

	AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
	TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());

	OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
	AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);

	// --------------- the tail (solution set join) ---------------
	AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph, numSubTasks);
	TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
	{
		tailConfig.setIterationId(ITERATION_ID);

		tailConfig.setIsWorksetIteration();
		tailConfig.setIsWorksetUpdate();

		tailConfig.setIsSolutionSetUpdate();
		tailConfig.setIsSolutionSetUpdateWithoutReprobe();

		// inputs and driver
		tailConfig.addInputToGroup(0);
		tailConfig.setInputSerializer(serializer, 0);

		// output
		tailConfig.setOutputSerializer(serializer);

		// the driver
		tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
		tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
		tailConfig.setDriverComparator(comparator, 0);
		tailConfig.setDriverPairComparator(pairComparator);
		
		tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
	}

	// -- edges ------------------------------------------------------------------------------------------------
	JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

	JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
	intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);

	JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
	tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);

	JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

	JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	vertices.setSlotSharingGroup(sharingGroup);
	edges.setSlotSharingGroup(sharingGroup);
	head.setSlotSharingGroup(sharingGroup);
	intermediate.setSlotSharingGroup(sharingGroup);
	tail.setSlotSharingGroup(sharingGroup);
	output.setSlotSharingGroup(sharingGroup);
	sync.setSlotSharingGroup(sharingGroup);
	
	intermediate.setStrictlyCoLocatedWith(head);
	tail.setStrictlyCoLocatedWith(head);

	return jobGraph;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static JobGraph createJobGraph(String pointsPath, String centersPath, String resultPath, int numSubTasks, int numIterations) {

		// -- init -------------------------------------------------------------------------------------------------
		final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
		@SuppressWarnings("unchecked")
		final TypeComparatorFactory<?> int0Comparator = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class });

		JobGraph jobGraph = new JobGraph("KMeans Iterative");

		// -- vertices ---------------------------------------------------------------------------------------------
		InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
		InputFormatVertex centers = createCentersInput(jobGraph, centersPath, numSubTasks, serializer);
		
		AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer);
		AbstractJobVertex mapper = createMapper(jobGraph, numSubTasks, serializer, serializer, serializer, int0Comparator);
		
		AbstractJobVertex reducer = createReducer(jobGraph, numSubTasks, serializer, int0Comparator, serializer);
		
		AbstractJobVertex sync = createSync(jobGraph, numIterations, numSubTasks);
		
		OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);

		// -- edges ------------------------------------------------------------------------------------------------
		JobGraphUtils.connect(points, mapper, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(centers, head, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(head, mapper, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		new TaskConfig(mapper.getConfiguration()).setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);
		new TaskConfig(mapper.getConfiguration()).setInputCached(0, true);
		new TaskConfig(mapper.getConfiguration()).setRelativeInputMaterializationMemory(0,
				MEMORY_FRACTION_PER_CONSUMER);

		JobGraphUtils.connect(mapper, reducer, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
		new TaskConfig(reducer.getConfiguration()).setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);
		
		JobGraphUtils.connect(head, output, ChannelType.NETWORK, DistributionPattern.POINTWISE);
		
		JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

		// -- instance sharing -------------------------------------------------------------------------------------
		
		SlotSharingGroup sharingGroup = new SlotSharingGroup();
		
		points.setSlotSharingGroup(sharingGroup);
		centers.setSlotSharingGroup(sharingGroup);
		head.setSlotSharingGroup(sharingGroup);
		mapper.setSlotSharingGroup(sharingGroup);
		reducer.setSlotSharingGroup(sharingGroup);
		sync.setSlotSharingGroup(sharingGroup);
		output.setSlotSharingGroup(sharingGroup);
		
		mapper.setStrictlyCoLocatedWith(head);
		reducer.setStrictlyCoLocatedWith(head);

		return jobGraph;
	}