Java PartitionMethod类代码示例

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
private <P> PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, Partitioner<P> customPartitioner,
		TypeInformation<P> partitionerTypeInfo, DataDistribution distribution, String partitionLocationName) {
	super(input, input.getType());

	Preconditions.checkNotNull(pMethod);
	Preconditions.checkArgument(pKeys != null || pMethod == PartitionMethod.REBALANCE, "Partitioning requires keys");
	Preconditions.checkArgument(pMethod != PartitionMethod.CUSTOM || customPartitioner != null, "Custom partioning requires a partitioner.");
	Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition.");

	if (distribution != null) {
		Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified.");
		Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())),
				"The types of the flat key fields must be equal to the types of the fields of the distribution.");
	}

	if (customPartitioner != null) {
		pKeys.validateCustomPartitioner(customPartitioner, partitionerTypeInfo);
	}

	this.pMethod = pMethod;
	this.pKeys = pKeys;
	this.partitionLocationName = partitionLocationName;
	this.customPartitioner = customPartitioner;
	this.distribution = distribution;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
public PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys) {
	super(input, input.getType());

	if(pMethod == PartitionMethod.HASH && pKeys == null) {
		throw new IllegalArgumentException("Hash Partitioning requires keys");
	} else if(pMethod == PartitionMethod.RANGE) {
		throw new UnsupportedOperationException("Range Partitioning not yet supported");
	}
	
	if(pKeys instanceof Keys.ExpressionKeys<?> && !(input.getType() instanceof CompositeType) ) {
		throw new IllegalArgumentException("Hash Partitioning with key fields only possible on Composite-type DataSets");
	}
	
	this.pMethod = pMethod;
	this.pKeys = pKeys;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
private static <T, K> MapOperatorBase<Tuple2<K, T>, T, ?> translateSelectorFunctionReducer(Keys.SelectorFunctionKeys<T, ?> rawKeys,
		PartitionMethod pMethod, TypeInformation<T> inputType, String name, Operator<T> input, int partitionDop)
{
	@SuppressWarnings("unchecked")
	final Keys.SelectorFunctionKeys<T, K> keys = (Keys.SelectorFunctionKeys<T, K>) rawKeys;
	
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, T>>(keys.getKeyType(), inputType);
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>>(typeInfoWithKey, typeInfoWithKey);
	
	KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper<T, K>(keys.getKeyExtractor());
	
	MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> keyExtractingMap = new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>(extractor, new UnaryOperatorInformation<T, Tuple2<K, T>>(inputType, typeInfoWithKey), "Key Extractor");
	PartitionOperatorBase<Tuple2<K, T>> noop = new PartitionOperatorBase<Tuple2<K, T>>(operatorInfo, pMethod, new int[]{0}, name);
	MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> keyRemovingMap = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<Tuple2<K, T>, T>(typeInfoWithKey, inputType), "Key Extractor");

	keyExtractingMap.setInput(input);
	noop.setInput(keyExtractingMap);
	keyRemovingMap.setInput(noop);
	
	// set dop
	keyExtractingMap.setDegreeOfParallelism(input.getDegreeOfParallelism());
	noop.setDegreeOfParallelism(partitionDop);
	keyRemovingMap.setDegreeOfParallelism(partitionDop);
	
	return keyRemovingMap;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Sets the order of keys for range partitioning.
 * NOTE: Only valid for {@link PartitionMethod#RANGE}.
 *
 * @param orders array of orders for each specified partition key
 * @return The partitioneOperator with properly set orders for given keys
    */
@PublicEvolving
public PartitionOperator<T> withOrders(Order... orders) {
	Preconditions.checkState(pMethod == PartitionMethod.RANGE, "Orders cannot be applied for %s partition " +
			"method", pMethod);
	Preconditions.checkArgument(pKeys.getOriginalKeyFieldTypes().length == orders.length, "The number of key " +
			"fields and orders should be the same.");
	this.orders = orders;

	return this;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Ordering ordering, Partitioner<?>
		customPartitioner, DataDistribution distribution) {
	super(pKeys);

	Preconditions.checkArgument(pMethod != PartitionMethod.RANGE
			|| pKeys.equals(new FieldSet(ordering.getFieldPositions())),
			"Partition keys must match the given ordering.");

	this.pMethod = pMethod;
	this.customPartitioner = customPartitioner;
	this.distribution = distribution;
	this.ordering = ordering;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys) {
	this.pMethod = pMethod;
	this.pKeys = pKeys;
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Hash-partitions a DataSet on the specified key fields.
 *
 * <p><b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 *
 * @param fields The field indexes on which the DataSet is hash-partitioned.
 * @return The partitioned DataSet.
 */
public PartitionOperator<T> partitionByHash(int... fields) {
	return new PartitionOperator<>(this, PartitionMethod.HASH, new Keys.ExpressionKeys<>(fields, getType()), Utils.getCallLocationName());
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Range-partitions a DataSet on the specified key fields.
 *
 * <p><b>Important:</b>This operation requires an extra pass over the DataSet to compute the range boundaries and
 * shuffles the whole DataSet over the network. This can take significant amount of time.
 *
 * @param fields The field indexes on which the DataSet is range-partitioned.
 * @return The partitioned DataSet.
 */
public PartitionOperator<T> partitionByRange(int... fields) {
	return new PartitionOperator<>(this, PartitionMethod.RANGE, new Keys.ExpressionKeys<>(fields, getType()), Utils.getCallLocationName());
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Enforces a re-balancing of the DataSet, i.e., the DataSet is evenly distributed over all parallel instances of the
 * following task. This can help to improve performance in case of heavy data skew and compute intensive operations.
 *
 * <p><b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 *
 * @return The re-balanced DataSet.
 */
public PartitionOperator<T> rebalance() {
	return new PartitionOperator<>(this, PartitionMethod.REBALANCE, Utils.getCallLocationName());
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Hash-partitions a DataSet on the specified key fields.
 * <p>
 * <b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 * 
 * @param fields The field indexes on which the DataSet is hash-partitioned.
 * @return The partitioned DataSet.
 */
public PartitionOperator<T> partitionByHash(int... fields) {
	return new PartitionOperator<T>(this, PartitionMethod.HASH, new Keys.ExpressionKeys<T>(fields, getType(), false));
}
 

import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod; //导入依赖的package包/类
/**
 * Enforces a rebalancing of the DataSet, i.e., the DataSet is evenly distributed over all parallel instances of the 
 * following task. This can help to improve performance in case of heavy data skew and compute intensive operations.
 * <p>
 * <b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
 * 
 * @return The rebalanced DataSet.
 */
public PartitionOperator<T> rebalance() {
	return new PartitionOperator<T>(this, PartitionMethod.REBALANCE);
}