Java TaskConfig.getNumOutputs方法代碼示例

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable containingTask, ClassLoader cl, TaskConfig config,
									List<ChainedDriver<?, ?>> chainedTasksTarget,
									List<RecordWriter<?>> eventualOutputs,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?,?>> accumulatorMap)
throws Exception
{
	final int numOutputs = config.getNumOutputs();

	// check whether we got any chained tasks
	final int numChained = config.getNumberOfChainedStubs();
	if (numChained > 0) {
		// got chained stubs. that means that this one may only have a single forward connection
		if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) {
			throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection.");
		}

		// instantiate each task
		@SuppressWarnings("rawtypes")
		Collector previous = null;
		for (int i = numChained - 1; i >= 0; --i)
		{
			// get the task first
			final ChainedDriver<?, ?> ct;
			try {
				Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i);
				ct = ctc.newInstance();
			}
			catch (Exception ex) {
				throw new RuntimeException("Could not instantiate chained task driver.", ex);
			}

			// get the configuration for the task
			final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
			final String taskName = config.getChainedTaskName(i);

			if (i == numChained - 1) {
				// last in chain, instantiate the output collector for this task
				previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
			}

			ct.setup(chainedStubConf, taskName, previous, containingTask, cl, executionConfig, accumulatorMap);
			chainedTasksTarget.add(0, ct);

			if (i == numChained - 1) {
				ct.getIOMetrics().reuseOutputMetricsForTask();
			}

			previous = ct;
		}
		// the collector of the first in the chain is the collector for the task
		return (Collector<T>) previous;
	}
	// else

	// instantiate the output collector the default way from this configuration
	return getOutputCollector(containingTask , config, cl, eventualOutputs, 0, numOutputs);
}
 

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
/**
	 * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
	 * channel is then the channel into the union node, the local strategy channel the one from the union to the
	 * actual target operator.
	 *
	 * @param channel
	 * @param inputNumber
	 * @param sourceVertex
	 * @param sourceConfig
	 * @param targetVertex
	 * @param targetConfig
	 * @param isBroadcast
	 * @throws CompilerException
	 */
	private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
			final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
			final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
	throws CompilerException
	{
		// ------------ connect the vertices to the job graph --------------
		final DistributionPattern distributionPattern;

		switch (channel.getShipStrategy()) {
			case FORWARD:
				distributionPattern = DistributionPattern.POINTWISE;
				break;
			case PARTITION_RANDOM:
			case BROADCAST:
			case PARTITION_HASH:
			case PARTITION_RANGE:
			case PARTITION_FORCED_REBALANCE:
				distributionPattern = DistributionPattern.BIPARTITE;
				break;
			default:
				throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
		}
		
		targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern);
		
//		sourceVertex.conn/ectTo(targetVertex, channelType, distributionPattern);

		// -------------- configure the source task's ship strategy strategies in task config --------------
		final int outputIndex = sourceConfig.getNumOutputs();
		sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
		if (outputIndex == 0) {
			sourceConfig.setOutputSerializer(channel.getSerializer());
		}
		if (channel.getShipStrategyComparator() != null) {
			sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
		}
		
		if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			
			final DataDistribution dataDistribution = channel.getDataDistribution();
			if(dataDistribution != null) {
				sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
			} else {
				throw new RuntimeException("Range partitioning requires data distribution");
				// TODO: inject code and configuration for automatic histogram generation
			}
		}
//		if (targetContract instanceof GenericDataSink) {
//			final DataDistribution distri = ((GenericDataSink) targetContract).getDataDistribution();
//			if (distri != null) {
//				configForOutputShipStrategy.setOutputDataDistribution(distri);
//			}
//		}
		
		// ---------------- configure the receiver -------------------
		if (isBroadcast) {
			targetConfig.addBroadcastInputToGroup(inputNumber);
		} else {
			targetConfig.addInputToGroup(inputNumber);
		}
		return distributionPattern;
	}
 

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable nepheleTask, ClassLoader cl, TaskConfig config,
				List<ChainedDriver<?, ?>> chainedTasksTarget, List<BufferWriter> eventualOutputs)
throws Exception
{
	final int numOutputs = config.getNumOutputs();

	// check whether we got any chained tasks
	final int numChained = config.getNumberOfChainedStubs();
	if (numChained > 0) {
		// got chained stubs. that means that this one may only have a single forward connection
		if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) {
			throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection.");
		}

		// instantiate each task
		@SuppressWarnings("rawtypes")
		Collector previous = null;
		for (int i = numChained - 1; i >= 0; --i)
		{
			// get the task first
			final ChainedDriver<?, ?> ct;
			try {
				Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i);
				ct = ctc.newInstance();
			}
			catch (Exception ex) {
				throw new RuntimeException("Could not instantiate chained task driver.", ex);
			}

			// get the configuration for the task
			final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
			final String taskName = config.getChainedTaskName(i);

			if (i == numChained -1) {
				// last in chain, instantiate the output collector for this task
				previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, chainedStubConf.getNumOutputs());
			}

			ct.setup(chainedStubConf, taskName, previous, nepheleTask, cl);
			chainedTasksTarget.add(0, ct);

			previous = ct;
		}
		// the collector of the first in the chain is the collector for the nephele task
		return (Collector<T>) previous;
	}
	// else

	// instantiate the output collector the default way from this configuration
	return getOutputCollector(nepheleTask , config, cl, eventualOutputs, numOutputs);
}