Java Operator类代码示例

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
	SelectorFunctionKeys<IN, ?> rawGroupingKey,
	SelectorFunctionKeys<IN, ?> rawSortingKey,
	Ordering groupOrdering,
	GroupReduceFunction<IN, OUT> function,
	TypeInformation<OUT> outputType,
	String name,
	Operator<IN> input,
	boolean combinable) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedReduceGroupOperator<>(
			function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrdering);

	return reducer;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) {
	@SuppressWarnings("unchecked")
	BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd;
	IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead();
	BulkIterationBase<T> iterationOperator =
			new BulkIterationBase<>(new UnaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration");

	if (iterationHead.getParallelism() > 0) {
		iterationOperator.setParallelism(iterationHead.getParallelism());
	}

	translated.put(iterationHead, iterationOperator.getPartialSolution());

	Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution());
	iterationOperator.setNextPartialSolution(translatedBody);
	iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations());
	iterationOperator.setInput(translate(iterationHead.getInput()));

	iterationOperator.getAggregators().addAll(iterationHead.getAggregators());

	if (iterationEnd.getTerminationCriterion() != null) {
		iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion()));
	}

	return iterationOperator;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
private void translateBcVariables(DataSet<?> setOrOp, Operator<?> dataFlowOp) {
	// check if this is actually an operator that could have broadcast variables
	if (setOrOp instanceof UdfOperator) {
		if (!(dataFlowOp instanceof AbstractUdfOperator<?, ?>)) {
			throw new RuntimeException("Error while creating the data flow plan for the program: A UDF operation was not translated to a UDF operator.");
		}

		UdfOperator<?> udfOp = (UdfOperator<?>) setOrOp;
		AbstractUdfOperator<?, ?> udfDataFlowOp = (AbstractUdfOperator<?, ?>) dataFlowOp;

		for (Map.Entry<String, DataSet<?>> bcVariable : udfOp.getBroadcastSets().entrySet()) {
			Operator<?> bcInput = translate(bcVariable.getValue());
			udfDataFlowOp.setBroadcastVariable(bcVariable.getKey(), bcInput);
		}
	}
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@Override
protected FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) {
	String name = getName() != null ? getName() : "FlatMap at " + defaultName;
	// create operator
	FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function,
		new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
	// set input
	po.setInput(input);
	// set parallelism
	if (this.getParallelism() > 0) {
		// use specified parallelism
		po.setParallelism(this.getParallelism());
	} else {
		// if no parallelism has been specified, use parallelism of input operator to enable chaining
		po.setParallelism(input.getParallelism());
	}

	return po;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@Override
protected MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) {

	String name = getName() != null ? getName() : "Map at " + defaultName;
	// create operator
	MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function,
			new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
	// set input
	po.setInput(input);
	// set parallelism
	if (this.getParallelism() > 0) {
		// use specified parallelism
		po.setParallelism(this.getParallelism());
	} else {
		// if no parallelism has been specified, use parallelism of input operator to enable chaining
		po.setParallelism(input.getParallelism());
	}

	return po;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingGroupCombineOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingGroupCombineOperator<IN, OUT, K> reducer =
		new PlanUnwrappingGroupCombineOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);

	return reducer;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
		SelectorFunctionKeys<IN, ?> rawGroupingKey,
		SelectorFunctionKeys<IN, ?> rawSortingKeys,
		Ordering groupOrder,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKeys;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedGroupCombineOperator<>(function, groupingKey, sortingKey, name, outputType, typeInfoWithKey);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrder);

	return reducer;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@Override
protected MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) {

	String name = getName() != null ? getName() : "MapPartition at " + defaultName;
	// create operator
	MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>> po = new MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
	// set input
	po.setInput(input);
	// set parallelism
	if (this.getParallelism() > 0) {
		// use specified parallelism
		po.setParallelism(this.getParallelism());
	} else {
		// if no parallelism has been specified, use parallelism of input operator to enable chaining
		po.setParallelism(input.getParallelism());
	}

	return po;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@Override
protected org.apache.flink.api.common.operators.base.FilterOperatorBase<T, FlatMapFunction<T, T>> translateToDataFlow(Operator<T> input) {

	String name = getName() != null ? getName() : "Filter at " + defaultName;

	// create operator
	PlanFilterOperator<T> po = new PlanFilterOperator<T>(function, name, getInputType());
	po.setInput(input);

	// set parallelism
	if (getParallelism() > 0) {
		// use specified parallelism
		po.setParallelism(getParallelism());
	} else {
		// if no parallelism has been specified, use parallelism of input operator to enable chaining
		po.setParallelism(input.getParallelism());
	}

	return po;
}
 

import org.apache.flink.api.common.operators.Operator; //导入依赖的package包/类
@Override
protected CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {

	String name = getName() != null ? getName() : "Cross at " + defaultName;
	// create operator
	CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> po =
			new CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>>(function,
					new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
					name);

	po.setFirstInput(input1);
	po.setSecondInput(input2);
	po.setParallelism(getParallelism());
	po.setCrossHint(hint);

	return po;
}