Java ParallelIteratorInputFormat类代码示例

import org.apache.flink.api.java.io.ParallelIteratorInputFormat; //导入依赖的package包/类
@Test
public void testReplicatedSourceToJoin() throws Exception {
	/*
	 * Test replicated source going into join
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>
			(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0l, 1000l))), BasicTypeInfo.LONG_TYPE_INFO)
			.map(new ToTuple());
	DataSet<Tuple1<Long>> source2 = env.generateSequence(0l, 1000l).map(new ToTuple());

	DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0)
			.projectFirst(0)
			.sum(0);

	List<Tuple> result = pairs.collect();

	String expectedResult = "(500500)";

	compareResultAsText(result, expectedResult);
}
 

import org.apache.flink.api.java.io.ParallelIteratorInputFormat; //导入依赖的package包/类
@Test
public void testReplicatedSourceToJoin() throws Exception {
	/*
	 * Test replicated source going into join
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<Long>> source1 = env.createInput(new ReplicatingInputFormat<Long, GenericInputSplit>
			(new ParallelIteratorInputFormat<Long>(new NumberSequenceIterator(0L, 1000L))), BasicTypeInfo.LONG_TYPE_INFO)
			.map(new ToTuple());
	DataSet<Tuple1<Long>> source2 = env.generateSequence(0L, 1000L).map(new ToTuple());

	DataSet<Tuple> pairs = source1.join(source2).where(0).equalTo(0)
			.projectFirst(0)
			.sum(0);

	List<Tuple> result = pairs.collect();

	String expectedResult = "(500500)";

	compareResultAsText(result, expectedResult);
}
 

import org.apache.flink.api.java.io.ParallelIteratorInputFormat; //导入依赖的package包/类
private <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type, String callLocationName) {
	return new DataSource<>(this, new ParallelIteratorInputFormat<>(iterator), type, callLocationName);
}
 

import org.apache.flink.api.java.io.ParallelIteratorInputFormat; //导入依赖的package包/类
/**
 * Creates a new data set that contains elements in the iterator. The iterator is splittable, allowing the
 * framework to create a parallel data source that returns the elements in the iterator.
 * The iterator must be serializable (as defined in {@link java.io.Serializable}, because the
 * execution environment may ship the elements into the cluster.
 * <p>
 * Because the iterator will remain unmodified until the actual execution happens, the type of data
 * returned by the iterator must be given explicitly in the form of the type information.
 * This method is useful for cases where the type is generic. In that case, the type class
 * (as given in {@link #fromParallelCollection(SplittableIterator, Class)} does not supply all type information.
 * 
 * @param iterator The iterator that produces the elements of the data set.
 * @param type The TypeInformation for the produced data set.
 * @return A DataSet representing the elements in the iterator.
 * 
 * @see #fromParallelCollection(SplittableIterator, Class)
 */
public <X> DataSource<X> fromParallelCollection(SplittableIterator<X> iterator, TypeInformation<X> type) {
	return new DataSource<X>(this, new ParallelIteratorInputFormat<X>(iterator), type);
}