Java InputFormatVertex.getConfiguration方法代码示例

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static InputFormatVertex createEdgesInput(JobGraph jobGraph, String edgesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// edges
	@SuppressWarnings("unchecked")
	CsvInputFormat edgesInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class);
	InputFormatVertex edgesInput = JobGraphUtils.createInput(edgesInFormat, edgesPath, "EdgesInput", jobGraph,
		numSubTasks);
	TaskConfig edgesInputConfig = new TaskConfig(edgesInput.getConfiguration());
	{
		edgesInputConfig.setOutputSerializer(serializer);
		edgesInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		edgesInputConfig.setOutputComparator(comparator, 0);
	}

	return edgesInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		
		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
	}

	return pointsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static InputFormatVertex createCentersInput(JobGraph jobGraph, String centersPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);

		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
	}

	return modelsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSourceTask.class);
	vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
	config.setStubParameters(node.getProgramOperator().getParameters());

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	vertex.setInvokableClass(DataSourceTask.class);

	// set user code
	config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
	config.setStubParameters(node.getPactContract().getParameters());

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static <T extends InputFormat<?,?>> InputFormatVertex createInput(UserCodeWrapper<T> stub, String name, JobGraph graph,
		int degreeOfParallelism)
{
	InputFormatVertex inputVertex = new InputFormatVertex(name);
	graph.addVertex(inputVertex);
	
	inputVertex.setInvokableClass(DataSourceTask.class);
	inputVertex.setParallelism(degreeOfParallelism);

	TaskConfig inputConfig = new TaskConfig(inputVertex.getConfiguration());
	inputConfig.setStubWrapper(stub);
	
	return inputVertex;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
private static InputFormatVertex createVerticesInput(JobGraph jobGraph, String verticesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	@SuppressWarnings("unchecked")
	CsvInputFormat verticesInFormat = new CsvInputFormat(' ', LongValue.class);
	InputFormatVertex verticesInput = JobGraphUtils.createInput(verticesInFormat, verticesPath, "VerticesInput",
		jobGraph, numSubTasks);
	TaskConfig verticesInputConfig = new TaskConfig(verticesInput.getConfiguration());
	{
		verticesInputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		verticesInputConfig.setOutputSerializer(serializer);

		// chained mapper that duplicates the id
		TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
		chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IdDuplicator>(IdDuplicator.class));
		chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		chainedMapperConfig.setInputSerializer(serializer, 0);

		chainedMapperConfig.setOutputSerializer(serializer);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.setOutputComparator(comparator, 0);
		chainedMapperConfig.setOutputComparator(comparator, 1);

		verticesInputConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "ID Duplicator");
	}

	return verticesInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat pointsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "Input[Points]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
	}

	return pointsInput;
}
 

import org.apache.flink.runtime.jobgraph.InputFormatVertex; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private static InputFormatVertex createModelsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat modelsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, pointsPath, "Input[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
		taskConfig.setOutputSerializer(serializer);
	}

	return modelsInput;
}