Java TypeSerializerFactory类代码示例

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
	this.inputIterators = new MutableObjectIterator<?>[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static InputFormatVertex createEdgesInput(JobGraph jobGraph, String edgesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// edges
	@SuppressWarnings("unchecked")
	CsvInputFormat edgesInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class);
	InputFormatVertex edgesInput = JobGraphUtils.createInput(edgesInFormat, edgesPath, "EdgesInput", jobGraph,
		numSubTasks);
	TaskConfig edgesInputConfig = new TaskConfig(edgesInput.getConfiguration());
	{
		edgesInputConfig.setOutputSerializer(serializer);
		edgesInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		edgesInputConfig.setOutputComparator(comparator, 0);
	}

	return edgesInput;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks,
		TypeSerializerFactory<?> serializer) {
	OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Final Output", numSubTasks);
	TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
	{

		outputConfig.addInputToGroup(0);
		outputConfig.setInputSerializer(serializer, 0);

		outputConfig.setStubWrapper(new UserCodeClassWrapper<CsvOutputFormat>(CsvOutputFormat.class));
		outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, resultPath);

		Configuration outputUserConfig = outputConfig.getStubParameters();
		outputUserConfig.setString(CsvOutputFormat.RECORD_DELIMITER_PARAMETER, "\n");
		outputUserConfig.setString(CsvOutputFormat.FIELD_DELIMITER_PARAMETER, " ");
		outputUserConfig.setClass(CsvOutputFormat.FIELD_TYPE_PARAMETER_PREFIX + 0, LongValue.class);
		outputUserConfig.setInteger(CsvOutputFormat.RECORD_POSITION_PARAMETER_PREFIX + 0, 0);
		outputUserConfig.setClass(CsvOutputFormat.FIELD_TYPE_PARAMETER_PREFIX + 1, LongValue.class);
		outputUserConfig.setInteger(CsvOutputFormat.RECORD_POSITION_PARAMETER_PREFIX + 1, 1);
		outputUserConfig.setInteger(CsvOutputFormat.NUM_FIELDS_PARAMETER, 2);
	}

	return output;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static AbstractJobVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
	AbstractJobVertex pointsInput = JobGraphUtils.createTask(RegularPactTask.class, "Map[DotProducts]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());

		taskConfig.setStubWrapper(new UserCodeClassWrapper<DotProducts>(DotProducts.class));
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		taskConfig.setDriver(CollectorMapDriver.class);
		taskConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);

		taskConfig.addInputToGroup(0);
		taskConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		taskConfig.setInputSerializer(serializer, 0);

		taskConfig.setBroadcastInputName("models", 0);
		taskConfig.addBroadcastInputToGroup(0);
		taskConfig.setBroadcastInputSerializer(serializer, 0);
	}

	return pointsInput;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
		taskConfig.addInputToGroup(0);
		taskConfig.setInputSerializer(serializer, 0);

		@SuppressWarnings("unchecked")
		CsvOutputFormat outFormat = new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class, LongValue.class);
		outFormat.setOutputFilePath(new Path(resultPath));
		
		taskConfig.setStubWrapper(new UserCodeObjectWrapper<CsvOutputFormat>(outFormat));
	}

	return output;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		
		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
	}

	return pointsInput;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static InputFormatVertex createCentersInput(JobGraph jobGraph, String centersPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);

		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
	}

	return modelsInput;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	
	OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
		taskConfig.addInputToGroup(0);
		taskConfig.setInputSerializer(serializer, 0);

		PointOutFormat outFormat = new PointOutFormat();
		outFormat.setOutputFilePath(new Path(resultPath));
		
		taskConfig.setStubWrapper(new UserCodeObjectWrapper<PointOutFormat>(outFormat));
	}

	return output;
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception {
	this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
	this.inputComparators = numComparators > 0 ? new TypeComparator[numComparators] : null;
	this.inputIterators = new MutableObjectIterator[numInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();
	
	for (int i = 0; i < numInputs; i++) {
		
		final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
		this.inputSerializers[i] = serializerFactory;
		
		this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
	}
	
	//  ---------------- create the driver's comparators ---------------------
	for (int i = 0; i < numComparators; i++) {
		
		if (this.inputComparators != null) {
			final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
			this.inputComparators[i] = comparatorFactory.createComparator();
		}
	}
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
private <BT> JoinHashMap<BT> initJoinHashMap() {
	TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer
			(getUserCodeClassLoader());
	TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator
			(getUserCodeClassLoader());

	TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer();
	TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator();

	return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator);
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
/**
 * @return output serializer of this task
 */
private TypeSerializer<OT> getOutputSerializer() {
	TypeSerializerFactory<OT> serializerFactory;

	if ((serializerFactory = getLastTasksConfig().getOutputSerializer(getUserCodeClassLoader())) ==
			null) {
		throw new RuntimeException("Missing output serializer for workset update.");
	}

	return serializerFactory.getSerializer();
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
/**
 * Creates all the serializers and iterators for the broadcast inputs.
 */
protected void initBroadcastInputsSerializers(int numBroadcastInputs) throws Exception {
	this.broadcastInputSerializers = new TypeSerializerFactory<?>[numBroadcastInputs];

	ClassLoader userCodeClassLoader = getUserCodeClassLoader();

	for (int i = 0; i < numBroadcastInputs; i++) {
		//  ---------------- create the serializer first ---------------------
		final TypeSerializerFactory<?> serializerFactory = this.config.getBroadcastInputSerializer(i, userCodeClassLoader);
		this.broadcastInputSerializers[i] = serializerFactory;
	}
}
 

import org.apache.flink.api.common.typeutils.TypeSerializerFactory; //导入依赖的package包/类
protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}