Java TaskConfig.setRelativeInputMaterializationMemory方法代碼示例

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
	// serializer
	if (isBroadcastChannel) {
		config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
		
		if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
			throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
		} else {
			return;
		}
	} else {
		config.setInputSerializer(channel.getSerializer(), inputNum);
	}
	
	// local strategy
	if (channel.getLocalStrategy() != LocalStrategy.NONE) {
		config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
		if (channel.getLocalStrategyComparator() != null) {
			config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
		}
	}
	
	assignLocalStrategyResources(channel, config, inputNum);
	
	// materialization / caching
	if (channel.getTempMode() != null) {
		final TempMode tm = channel.getTempMode();

		boolean needsMemory = false;
		// Don't add a pipeline breaker if the data exchange is already blocking, EXCEPT the channel is within an iteration.
		if (tm.breaksPipeline() &&
				(channel.isOnDynamicPath() || channel.getDataExchangeMode() != DataExchangeMode.BATCH) ) {
			config.setInputAsynchronouslyMaterialized(inputNum, true);
			needsMemory = true;
		}
		if (tm.isCached()) {
			config.setInputCached(inputNum, true);
			needsMemory = true;
		}
		
		if (needsMemory) {
			// sanity check
			if (tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
				throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
			}
			config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
		}
	}
}
 

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
	// serializer
	if (isBroadcastChannel) {
		config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
		
		if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
			throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
		} else {
			return;
		}
	} else {
		config.setInputSerializer(channel.getSerializer(), inputNum);
	}
	
	// local strategy
	if (channel.getLocalStrategy() != LocalStrategy.NONE) {
		config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
		if (channel.getLocalStrategyComparator() != null) {
			config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
		}
	}
	
	assignLocalStrategyResources(channel, config, inputNum);
	
	// materialization / caching
	if (channel.getTempMode() != null) {
		final TempMode tm = channel.getTempMode();

		boolean needsMemory = false;
		if (tm.breaksPipeline()) {
			config.setInputAsynchronouslyMaterialized(inputNum, true);
			needsMemory = true;
		}
		if (tm.isCached()) {
			config.setInputCached(inputNum, true);
			needsMemory = true;
		}
		
		if (needsMemory) {
			// sanity check
			if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
				throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
			}
			config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
		}
	}
}
 

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createIterationHead(JobGraph jobGraph, int numSubTasks,
		TypeSerializerFactory<?> serializer,
		TypeComparatorFactory<?> comparator,
		TypePairComparatorFactory<?, ?> pairComparator) {

	AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Join With Edges (Iteration Head)", jobGraph, numSubTasks);
	TaskConfig headConfig = new TaskConfig(head.getConfiguration());
	{
		headConfig.setIterationId(ITERATION_ID);

		// initial input / workset
		headConfig.addInputToGroup(0);
		headConfig.setInputSerializer(serializer, 0);
		headConfig.setInputComparator(comparator, 0);
		headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);

		// regular plan input (second input to the join)
		headConfig.addInputToGroup(1);
		headConfig.setInputSerializer(serializer, 1);
		headConfig.setInputComparator(comparator, 1);
		headConfig.setInputLocalStrategy(1, LocalStrategy.NONE);
		headConfig.setInputCached(1, true);
		headConfig.setRelativeInputMaterializationMemory(1, MEM_FRAC_PER_CONSUMER);

		// initial solution set input
		headConfig.addInputToGroup(2);
		headConfig.setInputSerializer(serializer, 2);
		headConfig.setInputComparator(comparator, 2);
		headConfig.setInputLocalStrategy(2, LocalStrategy.NONE);
		headConfig.setIterationHeadSolutionSetInputIndex(2);

		headConfig.setSolutionSetSerializer(serializer);
		headConfig.setSolutionSetComparator(comparator);

		// back channel / iterations
		headConfig.setIsWorksetIteration();
		headConfig.setRelativeBackChannelMemory(MEM_FRAC_PER_CONSUMER);
		headConfig.setRelativeSolutionSetMemory(MEM_FRAC_PER_CONSUMER );

		// output into iteration
		headConfig.setOutputSerializer(serializer);
		headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		headConfig.setOutputComparator(comparator, 0);

		// final output
		TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
		headFinalOutConfig.setOutputSerializer(serializer);
		headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);

		// the sync
		headConfig.setIterationHeadIndexOfSyncOutput(2);

		// the driver
		headConfig.setDriver(BuildSecondCachedMatchDriver.class);
		headConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
		headConfig.setStubWrapper(
			new UserCodeClassWrapper<NeighborWithComponentIDJoin>(NeighborWithComponentIDJoin.class));
		headConfig.setDriverComparator(comparator, 0);
		headConfig.setDriverComparator(comparator, 1);
		headConfig.setDriverPairComparator(pairComparator);
		headConfig.setRelativeMemoryDriver(MEM_FRAC_PER_CONSUMER);

		headConfig.addIterationAggregator(
			WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
	}

	return head;
}