Java DataDistribution类代码示例

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
private <P> PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, Partitioner<P> customPartitioner,
		TypeInformation<P> partitionerTypeInfo, DataDistribution distribution, String partitionLocationName) {
	super(input, input.getType());

	Preconditions.checkNotNull(pMethod);
	Preconditions.checkArgument(pKeys != null || pMethod == PartitionMethod.REBALANCE, "Partitioning requires keys");
	Preconditions.checkArgument(pMethod != PartitionMethod.CUSTOM || customPartitioner != null, "Custom partioning requires a partitioner.");
	Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition.");

	if (distribution != null) {
		Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified.");
		Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())),
				"The types of the flat key fields must be equal to the types of the fields of the distribution.");
	}

	if (customPartitioner != null) {
		pKeys.validateCustomPartitioner(customPartitioner, partitionerTypeInfo);
	}

	this.pMethod = pMethod;
	this.pKeys = pKeys;
	this.partitionLocationName = partitionLocationName;
	this.customPartitioner = customPartitioner;
	this.distribution = distribution;
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
/**
 * Creates a new channel selector that uses the given strategy (broadcasting, partitioning, ...)
 * and uses the supplied comparator to hash / compare records for partitioning them deterministically.
 * 
 * @param strategy The distribution strategy to be used.
 * @param comparator The comparator used to hash / compare the records.
 * @param distr The distribution pattern used in the case of a range partitioning.
 */
public OutputEmitter(ShipStrategyType strategy, TypeComparator<T> comparator, DataDistribution distr) {
	if (strategy == null) { 
		throw new NullPointerException();
	}
	
	this.strategy = strategy;
	this.comparator = comparator;
	
	switch (strategy) {
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANGE:
	case PARTITION_RANDOM:
	case PARTITION_FORCED_REBALANCE:
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}
	
	if ((strategy == ShipStrategyType.PARTITION_RANGE) && distr == null) {
		throw new NullPointerException("Data distribution must not be null when the ship strategy is range partitioning.");
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public OutputEmitter(ShipStrategyType strategy, int indexInSubtaskGroup, 
						TypeComparator<T> comparator, Partitioner<?> partitioner, DataDistribution distribution) {
	if (strategy == null) { 
		throw new NullPointerException();
	}

	this.strategy = strategy;
	this.nextChannelToSendTo = indexInSubtaskGroup;
	this.comparator = comparator;
	this.partitioner = (Partitioner<Object>) partitioner;
	this.distribution = distribution;


	switch (strategy) {
	case PARTITION_CUSTOM:
		extractedKeys = new Object[1];
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANDOM:
	case PARTITION_FORCED_REBALANCE:
		channels = new int[1];
		break;
	case PARTITION_RANGE:
		channels = new int[1];
		if (comparator != null) {
			this.flatComparators = comparator.getFlatComparators();
			this.keys = new Object[flatComparators.length];
		}
		break;
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}

	if (strategy == ShipStrategyType.PARTITION_CUSTOM && partitioner == null) {
		throw new NullPointerException("Partitioner must not be null when the ship strategy is set to custom partitioning.");
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) {
	this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName());
	
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
			DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) {
		
		distribution.write(out);
		config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray());
		
	}
	catch (IOException e) {
		throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e);
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public DataDistribution getOutputDataDistribution(int outputNum, final ClassLoader cl) throws ClassNotFoundException {
	final String className = this.config.getString(OUTPUT_DATA_DISTRIBUTION_CLASS, null);
	if (className == null) {
		return null;
	}
	
	final Class<? extends DataDistribution> clazz;
	try {
		clazz = Class.forName(className, true, cl).asSubclass(DataDistribution.class);
	} catch (ClassCastException ccex) {
		throw new CorruptConfigurationException("The class noted in the configuration as the data distribution " +
				"is no subclass of DataDistribution.");
	}
	
	final DataDistribution distribution = InstantiationUtil.instantiate(clazz, DataDistribution.class);
	
	final byte[] stateEncoded = this.config.getBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, null);
	if (stateEncoded == null) {
		throw new CorruptConfigurationException(
					"The configuration contained the data distribution type, but no serialized state.");
	}
	
	final ByteArrayInputStream bais = new ByteArrayInputStream(stateEncoded);
	final DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	
	try {
		distribution.read(in);
		return distribution;
	} catch (Exception ex) {
		throw new RuntimeException("The deserialization of the encoded data distribution state caused an error"
			+ (ex.getMessage() == null ? "." : ": " + ex.getMessage()), ex);
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
/**
 * Set the parameters for range partition.
 * 
 * @param ordering Order of the partitioned fields
 * @param distribution The data distribution for range partition. User can supply a customized data distribution,
 *                     also the data distribution can be null.  
 */
public void setRangePartitioned(Ordering ordering, DataDistribution distribution) {
	if (ordering == null) {
		throw new NullPointerException();
	}
	
	this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
	this.ordering = ordering;
	this.partitioningFields = ordering.getInvolvedIndexes();
	this.distribution = distribution;
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public void setRangePartitioned(Ordering ordering, DataDistribution dataDistribution) {
	if (ordering == null) {
		throw new NullPointerException();
	}
	this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
	this.ordering = ordering;
	this.partitioningFields = null;
	this.dataDistribution = dataDistribution;
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Ordering ordering, Partitioner<?>
		customPartitioner, DataDistribution distribution) {
	super(pKeys);

	Preconditions.checkArgument(pMethod != PartitionMethod.RANGE
			|| pKeys.equals(new FieldSet(ordering.getFieldPositions())),
			"Partition keys must match the given ordering.");

	this.pMethod = pMethod;
	this.customPartitioner = customPartitioner;
	this.distribution = distribution;
	this.ordering = ordering;
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
@Test
public void testRangePartitioningPreserved3() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	DataDistribution dd = new MockDistribution();
	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o, dd);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNotNull(filtered.getDataDistribution());
	assertEquals(dd, filtered.getDataDistribution());
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getCustomPartitioner());
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties iProps = new InterestingProperties();
	
	{
		final Ordering partitioning = getPactContract().getPartitionOrdering();
		final DataDistribution dataDist = getPactContract().getDataDistribution();
		final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
		if (partitioning != null) {
			if(dataDist != null) {
				partitioningProps.setRangePartitioned(partitioning, dataDist);
			} else {
				partitioningProps.setRangePartitioned(partitioning);
			}
			iProps.addGlobalProperties(partitioningProps);
		}
		iProps.addGlobalProperties(partitioningProps);
	}
	
	{
		final Ordering localOrder = getPactContract().getLocalOrder();
		final RequestedLocalProperties orderProps = new RequestedLocalProperties();
		if (localOrder != null) {
			orderProps.setOrdering(localOrder);
		}
		iProps.addLocalProperties(orderProps);
	}
	
	this.input.setInterestingProperties(iProps);
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
/**
 * Sets the sink to partition the records into ranges over the given ordering.
 * The bucket boundaries are determined using the given data distribution.
 * 
 * @param partitionOrdering The record ordering over which to partition in ranges.
 * @param distribution The distribution to use for the range partitioning.
 */
public void setRangePartitioned(Ordering partitionOrdering, DataDistribution distribution) {
	if (partitionOrdering.getNumberOfFields() != distribution.getNumberOfFields()) {
		throw new IllegalArgumentException("The number of keys in the distribution must match number of ordered fields.");
	}
	
	// TODO: check compatibility of distribution and ordering (number and order of keys, key types, etc.
	// TODO: adapt partition ordering to data distribution (use prefix of ordering)
	
	this.partitionOrdering = partitionOrdering;
	this.distribution = distribution;
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
/**
 * Creates a new channel selector that uses the given strategy (broadcasting, partitioning, ...)
 * and uses the supplied comparator to hash / compare records for partitioning them deterministically.
 * 
 * @param strategy The distribution strategy to be used.
 * @param comparator The comparator used to hash / compare the records.
 * @param distr The distribution pattern used in the case of a range partitioning.
 */
public RecordOutputEmitter(ShipStrategyType strategy, TypeComparator<Record> comparator, DataDistribution distr) {
	if (strategy == null) { 
		throw new NullPointerException();
	}
	
	this.strategy = strategy;
	this.comparator = comparator;
	this.distribution = distr;
	
	switch (strategy) {
	case FORWARD:
	case PARTITION_HASH:
	case PARTITION_RANGE:
	case PARTITION_RANDOM:
		this.channels = new int[1];
		break;
	case BROADCAST:
		break;
	default:
		throw new IllegalArgumentException("Invalid shipping strategy for OutputEmitter: " + strategy.name());
	}
	
	if ((strategy == ShipStrategyType.PARTITION_RANGE) && distr == null) {
		throw new NullPointerException("Data distribution must not be null when the ship strategy is range partitioning.");
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) {
	this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName());
	
	final ByteArrayOutputStream baos = new ByteArrayOutputStream();
	final DataOutputStream dos = new DataOutputStream(baos);
	try {
		distribution.write(new OutputViewDataOutputStreamWrapper(dos));
	} catch (IOException e) {
		throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e);
	}

	this.config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray());
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
public DataDistribution getOutputDataDistribution(int outputNum, final ClassLoader cl) throws ClassNotFoundException {
	final String className = this.config.getString(OUTPUT_DATA_DISTRIBUTION_CLASS, null);
	if (className == null) {
		return null;
	}
	
	final Class<? extends DataDistribution> clazz;
	try {
		clazz = (Class<? extends DataDistribution>) Class.forName(className, true, cl).asSubclass(DataDistribution.class);
	} catch (ClassCastException ccex) {
		throw new CorruptConfigurationException("The class noted in the configuration as the data distribution " +
				"is no subclass of DataDistribution.");
	}
	
	final DataDistribution distribution = InstantiationUtil.instantiate(clazz, DataDistribution.class);
	
	final byte[] stateEncoded = this.config.getBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, null);
	if (stateEncoded == null) {
		throw new CorruptConfigurationException(
					"The configuration contained the data distribution type, but no serialized state.");
	}
	
	final ByteArrayInputStream bais = new ByteArrayInputStream(stateEncoded);
	final DataInputStream in = new DataInputStream(bais);
	
	try {
		distribution.read(new InputViewDataInputStreamWrapper(in));
		return distribution;
	} catch (Exception ex) {
		throw new RuntimeException("The deserialization of the encoded data distribution state caused an error"
			+ ex.getMessage() == null ? "." : ": " + ex.getMessage(), ex);
	}
}
 

import org.apache.flink.api.common.distributions.DataDistribution; //导入依赖的package包/类
/**
 * Range-partitions a DataSet on the specified tuple field positions.
 */
public static <T> PartitionOperator<T> partitionByRange(DataSet<T> input, DataDistribution distribution, int... fields) {
	return new PartitionOperator<>(input, PartitionOperatorBase.PartitionMethod.RANGE, new Keys.ExpressionKeys<>(fields, input.getType(), false), distribution, Utils.getCallLocationName());
}