Java ShipStrategyType類代碼示例

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
public ShipStrategyType getOutputShipStrategy(int outputNum) {
	// check how many outputs are encoded in the config
	final int outputCnt = this.config.getInteger(OUTPUTS_NUM, -1);
	if (outputCnt < 1) {
		throw new CorruptConfigurationException("No output ship strategies are specified in the configuration.");
	}
	
	// sanity range checks
	if (outputNum < 0 || outputNum >= outputCnt) {
		throw new IllegalArgumentException("Invalid index for output shipping strategy.");
	}
	
	final int strategy = this.config.getInteger(OUTPUT_SHIP_STRATEGY_PREFIX + outputNum, -1);
	if (strategy == -1) {
		throw new CorruptConfigurationException("No output shipping strategy in configuration for output " + outputNum);
	} else if (strategy < 0 || strategy >= ShipStrategyType.values().length) {
		throw new CorruptConfigurationException("Illegal output shipping strategy in configuration for output "
																		+ outputNum + ": " + strategy);
	} else {
		return ShipStrategyType.values()[strategy];
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Test
public void testMissingKey() {
	// Test for IntValue
	@SuppressWarnings({"unchecked", "rawtypes"})
	final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {1}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
	
	Record rec = new Record(0);
	rec.setField(0, new IntValue(1));
	delegate.setInstance(rec);
	
	try {
		oe1.selectChannels(delegate, 100);
	} catch (KeyFieldOutOfBoundsException re) {
		Assert.assertEquals(1, re.getFieldNumber());
		return;
	}
	Assert.fail("Expected a KeyFieldOutOfBoundsException.");
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Test
public void testNullKey() {
	// Test for IntValue
	@SuppressWarnings({"unchecked", "rawtypes"})
	final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
	
	Record rec = new Record(2);
	rec.setField(1, new IntValue(1));
	delegate.setInstance(rec);

	try {
		oe1.selectChannels(delegate, 100);
	} catch (NullKeyFieldException re) {
		Assert.assertEquals(0, re.getFieldNumber());
		return;
	}
	Assert.fail("Expected a NullKeyFieldException.");
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
public static String getShipStrategyString(ShipStrategyType shipType) {
	if (shipType == null) {
		return "(null)";
	}
	switch (shipType) {
		case NONE:
			return "(none)";
		case FORWARD:
			return "Forward";
		case BROADCAST:
			return "Broadcast";
		case PARTITION_HASH:
			return "Hash Partition";
		case PARTITION_RANGE:
			return "Range Partition";
		case PARTITION_RANDOM:
			return "Redistribute";
		case PARTITION_FORCED_REBALANCE:
			return "Rebalance";
		case PARTITION_CUSTOM:
			return "Custom Partition";
		default:
			return shipType.name();
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
public void collect(Channel in, List<Channel> inputs) {
	if (in.getSource() instanceof NAryUnionPlanNode) {
		// sanity check
		if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
			throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
		}
		if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
			throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
		}

		inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
	} else {
		// is not a collapsed union node, so we take the channel directly
		inputs.add(in);
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy,
		FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders)
{
	super(template, nodeName, diverStrategy);
	this.input1 = input1;
	this.input2 = input2;
	this.keys1 = driverKeyFields1;
	this.keys2 = driverKeyFields2;
	this.sortOrders = driverSortOrders;
	
	if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input1.setReplicationFactor(getParallelism());
	}
	if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input2.setReplicationFactor(getParallelism());
	}
	
	mergeBranchPlanMaps(input1.getSource(), input2.getSource());
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner,
		SingleInputPlanNode reducer, SinkPlanNode sink) {
	// check ship strategies that are always fix
	Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy());
	Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());

	// check the driver strategies that are always fix
	Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy());
	Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy());
	Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
	if (combiner != null) {
		Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
		Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() &&
		ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNotNull("Plan should have a combiner", combiner);
		Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
		return true;
	} else {
		return false;
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
private boolean checkRepartitionShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.PARTITION_HASH == join.getInput1().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == join.getInput2().getShipStrategy() &&
		ShipStrategyType.FORWARD == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNull("Plan should not have a combiner", combiner);
		return true;
	} else {
		return false;
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Test
public void testMultiSolutionSetJoinPlan() {
	try {

		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		@SuppressWarnings("unchecked")
		DataSet<Tuple2<Long, Double>> inputData = env.fromElements(new Tuple2<Long, Double>(1L, 1.0));
		DataSet<Tuple2<Long, Double>> result = constructPlan(inputData, 10);

		// add two sinks, to test the case of branching after an iteration
		result.output(new DiscardingOutputFormat<Tuple2<Long, Double>>());
		result.output(new DiscardingOutputFormat<Tuple2<Long, Double>>());

		Plan p = env.createProgramPlan();

		OptimizedPlan optPlan = compileNoStats(p);

		OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);

		DualInputPlanNode join1 = or.getNode(JOIN_1);
		DualInputPlanNode join2 = or.getNode(JOIN_2);

		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, join1.getDriverStrategy());
		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, join2.getDriverStrategy());

		assertEquals(ShipStrategyType.PARTITION_HASH, join1.getInput2().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_HASH, join2.getInput1().getShipStrategy());

		assertEquals(SolutionSetPlanNode.class, join1.getInput1().getSource().getClass());
		assertEquals(SolutionSetPlanNode.class, join2.getInput2().getSource().getClass());

		new JobGraphGenerator().compileJobGraph(optPlan);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test erroneous: " + e.getMessage());
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");

		SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
		SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
		
		Channel toMap1 = new Channel(source1);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(source2);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		Channel toJoin1 = new Channel(map1);
		Channel toJoin2 = new Channel(map2);
		
		toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin1.setLocalStrategy(LocalStrategy.NONE);
		toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin2.setLocalStrategy(LocalStrategy.NONE);
		
		DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
		
		FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
		assertEquals(NO_PARTIAL_SOLUTION, report);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Test
public void testMultiKeys() {
	@SuppressWarnings({"unchecked", "rawtypes"})
	final TypeComparator<Record> multiComp = new RecordComparatorFactory(new int[] {0,1,3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, multiComp);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
	
	int numChannels = 100;
	int numRecords = 5000;
	
	int[] hit = new int[numChannels];

	for (int i = 0; i < numRecords; i++) {
		Record rec = new Record(4);
		rec.setField(0, new IntValue(i));
		rec.setField(1, new StringValue("AB"+i+"CD"+i));
		rec.setField(3, new DoubleValue(i*3.141d));
		delegate.setInstance(rec);
		
		int[] chans = oe1.selectChannels(delegate, hit.length);
		for (int chan : chans) {
			hit[chan]++;
		}
	}

	int cnt = 0;
	for (int aHit : hit) {
		assertTrue(aHit > 0);
		cnt += aHit;
	}
	assertTrue(cnt == numRecords);
	
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	Channel toReducer = in;

	if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
			(node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) {
		if (in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
	} else if (combinerStrategy != DriverStrategy.NONE) {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
							"Combine ("+node.getOperator().getName()+")", toCombiner,
							this.combinerStrategy, this.keyList);

		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());

		toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
	}

	return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer,
		DriverStrategy.SORTED_REDUCE, this.keyList);

}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_GROUP_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());

		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_GROUP_REDUCE);
	}
}
 

import org.apache.flink.runtime.operators.shipping.ShipStrategyType; //導入依賴的package包/類
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_REDUCE);
	}
}