當前位置: 首頁>>代碼示例>>Java>>正文

Java TaskConfig.addOutputShipStrategy方法代碼示例

本文整理匯總了Java中org.apache.flink.runtime.operators.util.TaskConfig.addOutputShipStrategy方法的典型用法代碼示例。如果您正苦於以下問題：Java TaskConfig.addOutputShipStrategy方法的具體用法？Java TaskConfig.addOutputShipStrategy怎麽用？Java TaskConfig.addOutputShipStrategy使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.flink.runtime.operators.util.TaskConfig的用法示例。


在下文中一共展示了TaskConfig.addOutputShipStrategy方法的13個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: createEdgesInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static InputFormatVertex createEdgesInput(JobGraph jobGraph, String edgesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// edges
	@SuppressWarnings("unchecked")
	CsvInputFormat edgesInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class);
	InputFormatVertex edgesInput = JobGraphUtils.createInput(edgesInFormat, edgesPath, "EdgesInput", jobGraph,
		numSubTasks);
	TaskConfig edgesInputConfig = new TaskConfig(edgesInput.getConfiguration());
	{
		edgesInputConfig.setOutputSerializer(serializer);
		edgesInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		edgesInputConfig.setOutputComparator(comparator, 0);
	}

	return edgesInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:18，代碼來源:ConnectedComponentsNepheleITCase.java

示例2: createMapper

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
	AbstractJobVertex pointsInput = JobGraphUtils.createTask(RegularPactTask.class, "Map[DotProducts]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());

		taskConfig.setStubWrapper(new UserCodeClassWrapper<DotProducts>(DotProducts.class));
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		taskConfig.setDriver(CollectorMapDriver.class);
		taskConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);

		taskConfig.addInputToGroup(0);
		taskConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		taskConfig.setInputSerializer(serializer, 0);

		taskConfig.setBroadcastInputName("models", 0);
		taskConfig.addBroadcastInputToGroup(0);
		taskConfig.setBroadcastInputSerializer(serializer, 0);
	}

	return pointsInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:24，代碼來源:BroadcastVarsNepheleITCase.java

示例3: createPointsInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
		
		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
	}

	return pointsInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:21，代碼來源:KMeansIterativeNepheleITCase.java

示例4: createCentersInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static InputFormatVertex createCentersInput(JobGraph jobGraph, String centersPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	@SuppressWarnings("unchecked")
	CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);

		TaskConfig chainedMapper = new TaskConfig(new Configuration());
		chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
		chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
		chainedMapper.setOutputSerializer(serializer);
		
		taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
	}

	return modelsInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:22，代碼來源:KMeansIterativeNepheleITCase.java

示例5: createVerticesInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static InputFormatVertex createVerticesInput(JobGraph jobGraph, String verticesPath, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	@SuppressWarnings("unchecked")
	CsvInputFormat verticesInFormat = new CsvInputFormat(' ', LongValue.class);
	InputFormatVertex verticesInput = JobGraphUtils.createInput(verticesInFormat, verticesPath, "VerticesInput",
		jobGraph, numSubTasks);
	TaskConfig verticesInputConfig = new TaskConfig(verticesInput.getConfiguration());
	{
		verticesInputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		verticesInputConfig.setOutputSerializer(serializer);

		// chained mapper that duplicates the id
		TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
		chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IdDuplicator>(IdDuplicator.class));
		chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
		chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		chainedMapperConfig.setInputSerializer(serializer, 0);

		chainedMapperConfig.setOutputSerializer(serializer);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		chainedMapperConfig.setOutputComparator(comparator, 0);
		chainedMapperConfig.setOutputComparator(comparator, 1);

		verticesInputConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "ID Duplicator");
	}

	return verticesInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:31，代碼來源:ConnectedComponentsNepheleITCase.java

示例6: createIterationIntermediate

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createIterationIntermediate(JobGraph jobGraph, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// --------------- the intermediate (reduce to min id) ---------------
	AbstractJobVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
		"Find Min Component-ID", jobGraph, numSubTasks);
	TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
	{
		intermediateConfig.setIterationId(ITERATION_ID);

		intermediateConfig.addInputToGroup(0);
		intermediateConfig.setInputSerializer(serializer, 0);
		intermediateConfig.setInputComparator(comparator, 0);
		intermediateConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
		intermediateConfig.setRelativeMemoryInput(0, MEM_FRAC_PER_CONSUMER);
		intermediateConfig.setFilehandlesInput(0, 64);
		intermediateConfig.setSpillingThresholdInput(0, 0.85f);

		intermediateConfig.setOutputSerializer(serializer);
		intermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);

		intermediateConfig.setDriver(GroupReduceDriver.class);
		intermediateConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		intermediateConfig.setDriverComparator(comparator, 0);
		intermediateConfig.setStubWrapper(
			new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingClassReduceFunction(MinimumComponentIDReduce.class)));
	}

	return intermediate;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:31，代碼來源:ConnectedComponentsNepheleITCase.java

示例7: createPointsInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@SuppressWarnings("unchecked")
private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat pointsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "Input[Points]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serializer);
	}

	return pointsInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:14，代碼來源:BroadcastVarsNepheleITCase.java

示例8: createModelsInput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@SuppressWarnings("unchecked")
private static InputFormatVertex createModelsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
	CsvInputFormat modelsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
	InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, pointsPath, "Input[Models]", jobGraph, numSubTasks);

	{
		TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
		taskConfig.setOutputSerializer(serializer);
	}

	return modelsInput;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:14，代碼來源:BroadcastVarsNepheleITCase.java

示例9: createIterationHead

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createIterationHead(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
	AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks);

	TaskConfig headConfig = new TaskConfig(head.getConfiguration());
	headConfig.setIterationId(ITERATION_ID);
	
	// initial input / partial solution
	headConfig.addInputToGroup(0);
	headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
	headConfig.setInputSerializer(serializer, 0);
	
	// back channel / iterations
	headConfig.setRelativeBackChannelMemory(MEMORY_FRACTION_PER_CONSUMER);
	
	// output into iteration. broadcasting the centers
	headConfig.setOutputSerializer(serializer);
	headConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
	
	// final output
	TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
	headFinalOutConfig.setOutputSerializer(serializer);
	headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
	
	// the sync
	headConfig.setIterationHeadIndexOfSyncOutput(2);
	
	// the driver 
	headConfig.setDriver(NoOpDriver.class);
	headConfig.setDriverStrategy(DriverStrategy.UNARY_NO_OP);
	
	return head;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:34，代碼來源:KMeansIterativeNepheleITCase.java

示例10: createMapper

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> inputSerializer,
		TypeSerializerFactory<?> broadcastVarSerializer, TypeSerializerFactory<?> outputSerializer,
		TypeComparatorFactory<?> outputComparator)
{
	AbstractJobVertex mapper = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
		"Map (Select nearest center)", jobGraph, numSubTasks);
	
	TaskConfig intermediateConfig = new TaskConfig(mapper.getConfiguration());
	intermediateConfig.setIterationId(ITERATION_ID);
	
	intermediateConfig.setDriver(CollectorMapDriver.class);
	intermediateConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
	intermediateConfig.addInputToGroup(0);
	intermediateConfig.setInputSerializer(inputSerializer, 0);
	
	intermediateConfig.setOutputSerializer(outputSerializer);
	intermediateConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
	intermediateConfig.setOutputComparator(outputComparator, 0);

	intermediateConfig.setBroadcastInputName("centers", 0);
	intermediateConfig.addBroadcastInputToGroup(0);
	intermediateConfig.setBroadcastInputSerializer(broadcastVarSerializer, 0);
	
	// the udf
	intermediateConfig.setStubWrapper(new UserCodeObjectWrapper<SelectNearestCenter>(new SelectNearestCenter()));
	
	return mapper;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:29，代碼來源:KMeansIterativeNepheleITCase.java

示例11: addOutput

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
public void addOutput(List<Record> output) {
	this.mockEnv.addOutput(output);
	TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
	conf.addOutputShipStrategy(ShipStrategyType.FORWARD);
	conf.setOutputSerializer(RecordSerializerFactory.get());
}
開發者ID:axbaretto，項目名稱:flink，代碼行數:7，代碼來源:TaskTestBase.java

示例12: connectJobVertices

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
/**
	 * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
	 * channel is then the channel into the union node, the local strategy channel the one from the union to the
	 * actual target operator.
	 *
	 * @param channel
	 * @param inputNumber
	 * @param sourceVertex
	 * @param sourceConfig
	 * @param targetVertex
	 * @param targetConfig
	 * @param isBroadcast
	 * @throws CompilerException
	 */
	private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
			final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
			final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
	throws CompilerException
	{
		// ------------ connect the vertices to the job graph --------------
		final DistributionPattern distributionPattern;

		switch (channel.getShipStrategy()) {
			case FORWARD:
				distributionPattern = DistributionPattern.POINTWISE;
				break;
			case PARTITION_RANDOM:
			case BROADCAST:
			case PARTITION_HASH:
			case PARTITION_RANGE:
			case PARTITION_FORCED_REBALANCE:
				distributionPattern = DistributionPattern.BIPARTITE;
				break;
			default:
				throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
		}
		
		targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern);
		
//		sourceVertex.conn/ectTo(targetVertex, channelType, distributionPattern);

		// -------------- configure the source task's ship strategy strategies in task config --------------
		final int outputIndex = sourceConfig.getNumOutputs();
		sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
		if (outputIndex == 0) {
			sourceConfig.setOutputSerializer(channel.getSerializer());
		}
		if (channel.getShipStrategyComparator() != null) {
			sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
		}
		
		if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			
			final DataDistribution dataDistribution = channel.getDataDistribution();
			if(dataDistribution != null) {
				sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
			} else {
				throw new RuntimeException("Range partitioning requires data distribution");
				// TODO: inject code and configuration for automatic histogram generation
			}
		}
//		if (targetContract instanceof GenericDataSink) {
//			final DataDistribution distri = ((GenericDataSink) targetContract).getDataDistribution();
//			if (distri != null) {
//				configForOutputShipStrategy.setOutputDataDistribution(distri);
//			}
//		}
		
		// ---------------- configure the receiver -------------------
		if (isBroadcast) {
			targetConfig.addBroadcastInputToGroup(inputNumber);
		} else {
			targetConfig.addInputToGroup(inputNumber);
		}
		return distributionPattern;
	}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:77，代碼來源:NepheleJobGraphGenerator.java

示例13: createIterationHead

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
private static AbstractJobVertex createIterationHead(JobGraph jobGraph, int numSubTasks,
		TypeSerializerFactory<?> serializer,
		TypeComparatorFactory<?> comparator,
		TypePairComparatorFactory<?, ?> pairComparator) {

	AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Join With Edges (Iteration Head)", jobGraph, numSubTasks);
	TaskConfig headConfig = new TaskConfig(head.getConfiguration());
	{
		headConfig.setIterationId(ITERATION_ID);

		// initial input / workset
		headConfig.addInputToGroup(0);
		headConfig.setInputSerializer(serializer, 0);
		headConfig.setInputComparator(comparator, 0);
		headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
		headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);

		// regular plan input (second input to the join)
		headConfig.addInputToGroup(1);
		headConfig.setInputSerializer(serializer, 1);
		headConfig.setInputComparator(comparator, 1);
		headConfig.setInputLocalStrategy(1, LocalStrategy.NONE);
		headConfig.setInputCached(1, true);
		headConfig.setRelativeInputMaterializationMemory(1, MEM_FRAC_PER_CONSUMER);

		// initial solution set input
		headConfig.addInputToGroup(2);
		headConfig.setInputSerializer(serializer, 2);
		headConfig.setInputComparator(comparator, 2);
		headConfig.setInputLocalStrategy(2, LocalStrategy.NONE);
		headConfig.setIterationHeadSolutionSetInputIndex(2);

		headConfig.setSolutionSetSerializer(serializer);
		headConfig.setSolutionSetComparator(comparator);

		// back channel / iterations
		headConfig.setIsWorksetIteration();
		headConfig.setRelativeBackChannelMemory(MEM_FRAC_PER_CONSUMER);
		headConfig.setRelativeSolutionSetMemory(MEM_FRAC_PER_CONSUMER );

		// output into iteration
		headConfig.setOutputSerializer(serializer);
		headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
		headConfig.setOutputComparator(comparator, 0);

		// final output
		TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
		headFinalOutConfig.setOutputSerializer(serializer);
		headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);

		// the sync
		headConfig.setIterationHeadIndexOfSyncOutput(2);

		// the driver
		headConfig.setDriver(BuildSecondCachedMatchDriver.class);
		headConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
		headConfig.setStubWrapper(
			new UserCodeClassWrapper<NeighborWithComponentIDJoin>(NeighborWithComponentIDJoin.class));
		headConfig.setDriverComparator(comparator, 0);
		headConfig.setDriverComparator(comparator, 1);
		headConfig.setDriverPairComparator(pairComparator);
		headConfig.setRelativeMemoryDriver(MEM_FRAC_PER_CONSUMER);

		headConfig.addIterationAggregator(
			WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
	}

	return head;
}
開發者ID:citlab，項目名稱:vs.msc.ws14，代碼行數:71，代碼來源:ConnectedComponentsNepheleITCase.java


注：本文中的org.apache.flink.runtime.operators.util.TaskConfig.addOutputShipStrategy方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。