Java ExecutionConfig.isObjectReuseEnabled方法代码示例

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
		throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
	}
	
	TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
		throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
	}
	this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
		throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
	}
	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.comparator = this.taskContext.getDriverComparator(0);
	this.input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
public ExecutionConfigSummary(ExecutionConfig ec) {
	executionMode = ec.getExecutionMode().name();
	if (ec.getRestartStrategy() != null) {
		restartStrategyDescription = ec.getRestartStrategy().getDescription();
	} else {
		restartStrategyDescription = "default";
	}
	parallelism = ec.getParallelism();
	objectReuseEnabled = ec.isObjectReuseEnabled();
	if (ec.getGlobalJobParameters() != null
			&& ec.getGlobalJobParameters().toMap() != null) {
		globalJobParameters = ec.getGlobalJobParameters().toMap();
	} else {
		globalJobParameters = Collections.emptyMap();
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	this.strategy = config.getDriverStrategy();

	switch (this.strategy) {
		case ALL_GROUP_REDUCE_COMBINE:
			if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
				throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
			}
		case ALL_GROUP_REDUCE:
		case ALL_GROUP_COMBINE:
			break;
		default:
			throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
	}

	this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
	this.input = this.taskContext.getInput(0);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("FlatMapDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("NoOpDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() {
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("MapPartitionDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void setup(TaskContext<MapFunction<IT, OT>, OT> context) {
	this.taskContext = context;
	this.running = true;

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if(driverStrategy != DriverStrategy.ALL_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for " +
				"GroupCombine.");
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
		AbstractInvokable parent, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
		Map<String, Accumulator<?,?>> accumulatorMap)
{
	this.config = config;
	this.taskName = taskName;
	this.userCodeClassLoader = userCodeClassLoader;
	this.metrics = parent.getEnvironment().getMetricGroup().addOperator(taskName);
	this.numRecordsIn = this.metrics.getIOMetricGroup().getNumRecordsInCounter();
	this.numRecordsOut = this.metrics.getIOMetricGroup().getNumRecordsOutCounter();
	this.outputCollector = new CountingCollector<>(outputCollector, numRecordsOut);

	Environment env = parent.getEnvironment();

	if (parent instanceof BatchTask) {
		this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
	} else {
		this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
				parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics
		);
	}

	this.executionConfig = executionConfig;
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	setup(parent);
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	};
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
	
	final TypeSerializer<IT2> solutionSetSerializer;
	final TypeComparator<IT2> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		Object table = SolutionSetBroker.instance().get(identifier);
		
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT2>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT2>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	}
	else {
		throw new Exception("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
	
	TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); 
	
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
		probeSideRecord = probeSideSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
 

import org.apache.flink.api.common.ExecutionConfig; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void initialize() {
	
	final TypeComparator<IT1> solutionSetComparator;
	
	// grab a handle to the hash table from the iteration broker
	if (taskContext instanceof AbstractIterativeTask) {
		AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
		String identifier = iterativeTaskContext.brokerKey();
		
		Object table = SolutionSetBroker.instance().get(identifier);
		if (table instanceof CompactingHashTable) {
			this.hashTable = (CompactingHashTable<IT1>) table;
			solutionSetSerializer = this.hashTable.getBuildSideSerializer();
			solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
		}
		else if (table instanceof JoinHashMap) {
			this.objectMap = (JoinHashMap<IT1>) table;
			solutionSetSerializer = this.objectMap.getBuildSerializer();
			solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
		}
		else {
			throw new RuntimeException("Unrecognized solution set index: " + table);
		}
	} else {
		throw new RuntimeException("The task context of this driver is no iterative task context.");
	}
	
	TaskConfig config = taskContext.getTaskConfig();
	ClassLoader classLoader = taskContext.getUserCodeClassLoader();
	
	TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
	
	this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
	this.probeSideComparator = probeSideComparatorFactory.createComparator();
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (objectReuseEnabled) {
		solutionSideRecord = solutionSetSerializer.createInstance();
	}
	
	TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
	pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}