Java UnilateralSortMerger類代碼示例

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
protected DriverTestBase(ExecutionConfig executionConfig, long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManagerAsync();
	this.memManager = totalMem > 0 ? new MemoryManager(totalMem,1) : null;

	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	this.taskConfig = new TaskConfig(new Configuration());
	this.executionConfig = executionConfig;
	this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
@SuppressWarnings("unchecked")
public void addInputSorted(MutableObjectIterator<IN> input, TypeSerializer<IN> serializer, TypeComparator<IN> comp) throws Exception {
	this.inputSerializers.add(serializer);
	UnilateralSortMerger<IN> sorter = new UnilateralSortMerger<>(
			this.memManager,
			this.ioManager,
			input,
			this.owner,
			new RuntimeSerializerFactory<>(serializer, (Class<IN>) serializer.createInstance().getClass()),
			comp,
			this.perSortFractionMem,
			32,
			0.8f,
			true /*use large record handler*/,
			false
	);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
@After
public void shutdownAll() throws Exception {
	// 1st, shutdown sorters
	for (UnilateralSortMerger<?> sorter : this.sorters) {
		if (sorter != null) {
			sorter.close();
		}
	}
	this.sorters.clear();
	
	// 2nd, shutdown I/O
	this.ioManager.shutdown();
	Assert.assertTrue("I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());
	
	// last, verify all memory is returned and shutdown mem manager
	MemoryManager memMan = getMemoryManager();
	if (memMan != null) {
		Assert.assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
		memMan.shutdown();
	}
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
protected DriverTestBase(long memory, int maxNumSorters, long perSortMemory) {
	if (memory < 0 || maxNumSorters < 0 || perSortMemory < 0) {
		throw new IllegalArgumentException();
	}
	
	final long totalMem = Math.max(memory, 0) + (Math.max(maxNumSorters, 0) * perSortMemory);
	
	this.perSortMem = perSortMemory;
	this.perSortFractionMem = (double)perSortMemory/totalMem;
	this.ioManager = new IOManager();
	this.memManager = totalMem > 0 ? new DefaultMemoryManager(totalMem,1) : null;
	
	this.inputs = new ArrayList<MutableObjectIterator<Record>>();
	this.comparators = new ArrayList<TypeComparator<Record>>();
	this.sorters = new ArrayList<UnilateralSortMerger<Record>>();
	
	this.owner = new DummyInvokable();
	
	this.config = new Configuration();
	this.taskConfig = new TaskConfig(this.config);
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
public void addInputSorted(MutableObjectIterator<Record> input, RecordComparator comp) throws Exception {
	UnilateralSortMerger<Record> sorter = new UnilateralSortMerger<Record>(
			this.memManager, this.ioManager, input, this.owner, RecordSerializerFactory.get(), comp,
			this.perSortFractionMem, 32, 0.8f, true /*use large record handler*/, true);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
public void addInputSorted(MutableObjectIterator<IN> input,
							TypeSerializer<IN> serializer,
							TypeComparator<IN> comp) throws Exception
{
	this.input = null;
	this.inputSerializer = serializer;
	this.sorter = new UnilateralSortMerger<IN>(
			this.memManager, this.ioManager, input, this.owner,
			this.<IN>getInputSerializer(0),
			comp,
			this.perSortFractionMem, 32, 0.8f,
			true /*use large record handler*/,
			false);
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
public void addInputSorted(MutableObjectIterator<Record> input, RecordComparator comp) throws Exception {
	UnilateralSortMerger<Record> sorter = new UnilateralSortMerger<Record>(
			this.memManager, this.ioManager, input, this.owner, RecordSerializerFactory.get(), comp,
			this.perSortFractionMem, 32, 0.8f);
	this.sorters.add(sorter);
	this.inputs.add(null);
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
@Test
public void testSortBothMerge() {
	try {
		
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter1 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, 
				this.comparator1.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final UnilateralSortMerger<Tuple2<Integer, String>> sorter2 = new UnilateralSortMerger<>(
				this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, 
				this.comparator2.duplicate(), (double)MEMORY_FOR_SORTER/MEMORY_SIZE, 128, 0.8f,
				true /*use large record handler*/, true);
		
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput1 = sorter1.getIterator();
		final MutableObjectIterator<Tuple2<Integer, String>> sortedInput2 = sorter2.getIterator();
		
		// compare with iterator values
		ReusingMergeInnerJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingMergeInnerJoinIterator<>(sortedInput1, sortedInput2,
					this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		sorter1.close();
		sorter2.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Sort-Merge Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum));
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
					e.getMessage() == null ? "." : ": " + e.getMessage(), e);
			}
			
			if (!(localStub instanceof FlatCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(FlatCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum));
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 

import org.apache.flink.runtime.operators.sort.UnilateralSortMerger; //導入依賴的package包/類
@Test
public void testSortBothMerge() {
	try {
		
		Generator generator1 = new Generator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		Generator generator2 = new Generator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.GeneratorIterator input1 = new TestData.GeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.GeneratorIterator input2 = new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);
		
		final JoinFunction matcher = new NoOpMatcher();
		final Collector<Record> collector = new DiscardingOutputCollector<Record>();
		
		long start = System.nanoTime();
		
		final UnilateralSortMerger<Record> sorter1 = new UnilateralSortMerger<Record>(
				this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, 
				this.comparator1.duplicate(), MEMORY_FOR_SORTER, 128, 0.8f);
		
		final UnilateralSortMerger<Record> sorter2 = new UnilateralSortMerger<Record>(
				this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, 
				this.comparator2.duplicate(), MEMORY_FOR_SORTER, 128, 0.8f);
		
		final MutableObjectIterator<Record> sortedInput1 = sorter1.getIterator();
		final MutableObjectIterator<Record> sortedInput2 = sorter2.getIterator();
		
		// compare with iterator values
		MergeMatchIterator<Record, Record, Record> iterator = 
			new MergeMatchIterator<Record, Record, Record>(sortedInput1, sortedInput2, 
					this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		sorter1.close();
		sorter2.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Sort-Merge Took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}