当前位置: 首页>>代码示例>>Java>>正文

Java GroupReduceDriver类代码示例

本文整理汇总了Java中org.apache.flink.runtime.operators.GroupReduceDriver类的典型用法代码示例。如果您正苦于以下问题：Java GroupReduceDriver类的具体用法？Java GroupReduceDriver怎么用？Java GroupReduceDriver使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。


GroupReduceDriver类属于org.apache.flink.runtime.operators包，在下文中一共展示了GroupReduceDriver类的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testAllReduceDriverImmutableEmpty

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Assert.assertTrue(result.getList().isEmpty());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:axbaretto，项目名称:flink，代码行数:32，代码来源:GroupReduceDriverTest.java

示例2: testAllReduceDriverImmutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(result);
		context.setComparator1(comparator);
		context.setUdf(new ConcatSumReducer());
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:axbaretto，项目名称:flink，代码行数:36，代码来源:GroupReduceDriverTest.java

示例3: testAllReduceDriverMutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverMutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableReducer());
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:axbaretto，项目名称:flink，代码行数:36，代码来源:GroupReduceDriverTest.java

示例4: testAllReduceDriverAccumulatingImmutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverAccumulatingImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableAccumulatingReducer());
		context.setMutableObjectMode(false);
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:axbaretto，项目名称:flink，代码行数:37，代码来源:GroupReduceDriverTest.java

示例5: createIterationIntermediate

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
private static AbstractJobVertex createIterationIntermediate(JobGraph jobGraph, int numSubTasks,
		TypeSerializerFactory<?> serializer, TypeComparatorFactory<?> comparator)
{
	// --------------- the intermediate (reduce to min id) ---------------
	AbstractJobVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
		"Find Min Component-ID", jobGraph, numSubTasks);
	TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
	{
		intermediateConfig.setIterationId(ITERATION_ID);

		intermediateConfig.addInputToGroup(0);
		intermediateConfig.setInputSerializer(serializer, 0);
		intermediateConfig.setInputComparator(comparator, 0);
		intermediateConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
		intermediateConfig.setRelativeMemoryInput(0, MEM_FRAC_PER_CONSUMER);
		intermediateConfig.setFilehandlesInput(0, 64);
		intermediateConfig.setSpillingThresholdInput(0, 0.85f);

		intermediateConfig.setOutputSerializer(serializer);
		intermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);

		intermediateConfig.setDriver(GroupReduceDriver.class);
		intermediateConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		intermediateConfig.setDriverComparator(comparator, 0);
		intermediateConfig.setStubWrapper(
			new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingClassReduceFunction(MinimumComponentIDReduce.class)));
	}

	return intermediate;
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:31，代码来源:ConnectedComponentsNepheleITCase.java

示例6: createReducer

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
private static AbstractJobVertex createReducer(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> inputSerializer,
		TypeComparatorFactory<?> inputComparator, TypeSerializerFactory<?> outputSerializer)
{
	// ---------------- the tail (reduce) --------------------
	
	AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "Reduce / Iteration Tail", jobGraph,
		numSubTasks);
	
	TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
	tailConfig.setIterationId(ITERATION_ID);
	tailConfig.setIsWorksetUpdate();
	
	// inputs and driver
	tailConfig.setDriver(GroupReduceDriver.class);
	tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	tailConfig.addInputToGroup(0);
	tailConfig.setInputSerializer(inputSerializer, 0);		
	tailConfig.setDriverComparator(inputComparator, 0);

	tailConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
	tailConfig.setInputComparator(inputComparator, 0);
	tailConfig.setRelativeMemoryInput(0, MEMORY_FRACTION_PER_CONSUMER);
	tailConfig.setFilehandlesInput(0, 128);
	tailConfig.setSpillingThresholdInput(0, 0.9f);
	
	// output
	tailConfig.setOutputSerializer(outputSerializer);
	
	// the udf
	tailConfig.setStubWrapper(new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingReduceFunction(new RecomputeClusterCenter())));
	
	return tail;
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:34，代码来源:KMeansIterativeNepheleITCase.java

示例7: testAllReduceDriverImmutableEmpty

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
		
		context.setInput1(input, typeInfo.createSerializer());
		context.setComparator1(comparator);
		context.setCollector(result);
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Assert.assertTrue(result.getList().isEmpty());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:32，代码来源:GroupReduceDriverTest.java

示例8: testAllReduceDriverImmutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer());
		context.setCollector(result);
		context.setComparator1(comparator);
		context.setUdf(new ConcatSumReducer());
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:36，代码来源:GroupReduceDriverTest.java

示例9: testAllReduceDriverMutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverMutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer());
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableReducer());
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:36，代码来源:GroupReduceDriverTest.java

示例10: testAllReduceDriverAccumulatingImmutable

import org.apache.flink.runtime.operators.GroupReduceDriver; //导入依赖的package包/类
@Test
public void testAllReduceDriverAccumulatingImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
		
		List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
		TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
		TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
		
		GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer());
		context.setComparator1(comparator);
		context.setCollector(result);
		context.setUdf(new ConcatSumMutableAccumulatingReducer());
		context.setMutableObjectMode(false);
		
		GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
开发者ID:citlab，项目名称:vs.msc.ws14，代码行数:37，代码来源:GroupReduceDriverTest.java


注：本文中的org.apache.flink.runtime.operators.GroupReduceDriver类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。