Java UserCodeObjectWrapper类代码示例

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
@Test
public void testInputFormatVertex() {
	try {
		final TestInputFormat inputFormat = new TestInputFormat();
		final InputFormatVertex vertex = new InputFormatVertex("Name");
		new TaskConfig(vertex.getConfiguration()).setStubWrapper(new UserCodeObjectWrapper<InputFormat<?, ?>>(inputFormat));
		
		final ClassLoader cl = getClass().getClassLoader();
		
		vertex.initializeOnMaster(cl);
		InputSplit[] splits = vertex.getInputSplitSource().createInputSplits(77);
		
		assertNotNull(splits);
		assertEquals(1, splits.length);
		assertEquals(TestSplit.class, splits[0].getClass());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
public void registerFileInputTask(AbstractInvokable inTask,
		Class<? extends DelimitedInputFormat<Record>> stubClass, String inPath, String delimiter)
{
	DelimitedInputFormat<Record> format;
	try {
		format = stubClass.newInstance();
	}
	catch (Throwable t) {
		throw new RuntimeException("Could not instantiate test input format.", t);
	}

	format.setFilePath(inPath);
	format.setDelimiter(delimiter);

	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());
	dsConfig.setStubWrapper(new UserCodeObjectWrapper<>(format));

	this.inputSplitProvider.addInputSplits(inPath, 5);
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
		taskConfig.addInputToGroup(0);
		taskConfig.setInputSerializer(serializer, 0);

		@SuppressWarnings("unchecked")
		CsvOutputFormat outFormat = new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class, LongValue.class);
		outFormat.setOutputFilePath(new Path(resultPath));
		
		taskConfig.setStubWrapper(new UserCodeObjectWrapper<CsvOutputFormat>(outFormat));
	}

	return output;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		
		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
	}

	return pointsInput;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static InputFormatVertex createCentersInput(JobGraph jobGraph, String centersPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);

		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
	}

	return modelsInput;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	
	OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
		taskConfig.addInputToGroup(0);
		taskConfig.setInputSerializer(serializer, 0);

		PointOutFormat outFormat = new PointOutFormat();
		outFormat.setOutputFilePath(new Path(resultPath));
		
		taskConfig.setStubWrapper(new UserCodeObjectWrapper<PointOutFormat>(outFormat));
	}

	return output;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
public void registerFileInputTask(AbstractInvokable inTask,
		Class<? extends DelimitedInputFormat> stubClass, String inPath, String delimiter)
{
	DelimitedInputFormat format;
	try {
		format = stubClass.newInstance();
	}
	catch (Throwable t) {
		throw new RuntimeException("Could not instantiate test input format.", t);
	}
	
	format.setFilePath(inPath);
	format.setDelimiter(delimiter);
	
	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());
	dsConfig.setStubWrapper(new UserCodeObjectWrapper<DelimitedInputFormat>(format));
	
	this.inputSplitProvider.addInputSplits(inPath, 5);

	inTask.setEnvironment(this.mockEnv);

	inTask.registerInputOutput();
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
/**
 * Creates a new instance for the given file using the given input format.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 * @param name The given name for the Pact, used in plans, logs and progress messages.
 */
public GenericDataSourceBase(T format, OperatorInformation<OUT> operatorInfo, String name) {
	super(operatorInfo, name);

	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}

	this.formatWrapper = new UserCodeObjectWrapper<T>(format);
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
public void registerFileOutputTask(FileOutputFormat<Record> outputFormat, String outPath) {
	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	dsConfig.setStubWrapper(new UserCodeObjectWrapper<>(outputFormat));
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static AbstractJobVertex createIterationIntermediate(JobGraph jobGraph, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// --------------- the intermediate (reduce to min id) ---------------
	AbstractJobVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
		"Find Min Component-ID", jobGraph, numSubTasks);
	TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
	{
		intermediateConfig.setIterationId(ITERATION_ID);

		intermediateConfig.addInputToGroup(0);
		intermediateConfig.setInputSerializer(serializer, 0);
		intermediateConfig.setInputComparator(comparator, 0);
		intermediateConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
		intermediateConfig.setRelativeMemoryInput(0, MEM_FRAC_PER_CONSUMER);
		intermediateConfig.setFilehandlesInput(0, 64);
		intermediateConfig.setSpillingThresholdInput(0, 0.85f);

		intermediateConfig.setOutputSerializer(serializer);
		intermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);

		intermediateConfig.setDriver(GroupReduceDriver.class);
		intermediateConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		intermediateConfig.setDriverComparator(comparator, 0);
		intermediateConfig.setStubWrapper(
			new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingClassReduceFunction(MinimumComponentIDReduce.class)));
	}

	return intermediate;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static AbstractJobVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> inputSerializer,
		TypeSerializerFactory<?> broadcastVarSerializer, TypeSerializerFactory<?> outputSerializer,
		TypeComparatorFactory<?> outputComparator)
{
	AbstractJobVertex mapper = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
		"Map (Select nearest center)", jobGraph, numSubTasks);
	
	TaskConfig intermediateConfig = new TaskConfig(mapper.getConfiguration());
	intermediateConfig.setIterationId(ITERATION_ID);
	
	intermediateConfig.setDriver(CollectorMapDriver.class);
	intermediateConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
	intermediateConfig.addInputToGroup(0);
	intermediateConfig.setInputSerializer(inputSerializer, 0);
	
	intermediateConfig.setOutputSerializer(outputSerializer);
	intermediateConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
	intermediateConfig.setOutputComparator(outputComparator, 0);

	intermediateConfig.setBroadcastInputName("centers", 0);
	intermediateConfig.addBroadcastInputToGroup(0);
	intermediateConfig.setBroadcastInputSerializer(broadcastVarSerializer, 0);
	
	// the udf
	intermediateConfig.setStubWrapper(new UserCodeObjectWrapper<SelectNearestCenter>(new SelectNearestCenter()));
	
	return mapper;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
private static AbstractJobVertex createReducer(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> inputSerializer,
		TypeComparatorFactory<?> inputComparator, TypeSerializerFactory<?> outputSerializer)
{
	// ---------------- the tail (reduce) --------------------
	
	AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "Reduce / Iteration Tail", jobGraph,
		numSubTasks);
	
	TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
	tailConfig.setIterationId(ITERATION_ID);
	tailConfig.setIsWorksetUpdate();
	
	// inputs and driver
	tailConfig.setDriver(GroupReduceDriver.class);
	tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	tailConfig.addInputToGroup(0);
	tailConfig.setInputSerializer(inputSerializer, 0);		
	tailConfig.setDriverComparator(inputComparator, 0);

	tailConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
	tailConfig.setInputComparator(inputComparator, 0);
	tailConfig.setRelativeMemoryInput(0, MEMORY_FRACTION_PER_CONSUMER);
	tailConfig.setFilehandlesInput(0, 128);
	tailConfig.setSpillingThresholdInput(0, 0.9f);
	
	// output
	tailConfig.setOutputSerializer(outputSerializer);
	
	// the udf
	tailConfig.setStubWrapper(new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingReduceFunction(new RecomputeClusterCenter())));
	
	return tail;
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
/**
 * Creates a Builder with the provided {@link ReduceFunction} implementation.
 * 
 * @param udf The {@link ReduceFunction} implementation for this Reduce contract.
 */
public static Builder builder(ReduceFunction udf) {
	UserCodeWrapper<ReduceFunction> original = new UserCodeObjectWrapper<ReduceFunction>(udf);
	UserCodeWrapper<GroupReduceFunction<Record, Record>> wrapped =
			new UserCodeObjectWrapper<GroupReduceFunction<Record, Record>>(new WrappingReduceFunction(udf));
	
	return new Builder(original, wrapped);
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
public InnerJoinOperatorBase(FT udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1,
		int[] keyPositions2, String name) {
	super(new UserCodeObjectWrapper<FT>(udf), operatorInfo, keyPositions1, keyPositions2, name);
}
 

import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper; //导入依赖的package包/类
public CoGroupRawOperatorBase(FT udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
	this(new UserCodeObjectWrapper<FT>(udf), operatorInfo, keyPositions1, keyPositions2, name);
}