Java FixedLengthRecordSorter类代码示例

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = config.getStubParameters();
	BatchTask.openUserCode(reducer, stubConfig);

	// instantiate the serializer / comparator
	serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer();
	comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator();

	MemoryManager memManager = parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
	memory = memManager.allocatePages(parent, numMemoryPages);

	LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");

	switch (strategy) {
		case SORTED_PARTIAL_REDUCE:
			// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
			if (comparator.supportsSerializationWithKeyNormalization() &&
				serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
				sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
			} else {
				sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
			}
			break;
		case HASHED_PARTIAL_REDUCE:
			table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
			table.open();
			reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled);
			break;
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void prepare() throws Exception {
	if (this.taskContext.getTaskConfig().getDriverStrategy() != DriverStrategy.SORTED_PARTIAL_REDUCE) {
		throw new Exception("Invalid strategy " + this.taskContext.getTaskConfig().getDriverStrategy() + " for reduce combiner.");
	}
	
	this.memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig()
			.getRelativeMemoryDriver());
	
	// instantiate the serializer / comparator
	final TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.comparator = this.taskContext.getDriverComparator(0);
	this.serializer = serializerFactory.getSerializer();
	this.reducer = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	final List<MemorySegment> memory = this.memManager.allocatePages(this.taskContext.getOwningNepheleTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (this.comparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<T>(this.serializer, this.comparator, memory);
	} else {
		this.sorter = new NormalizedKeySorter<T>(this.serializer, this.comparator.duplicate(), memory);
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	RegularPactTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the asynchronous sorter -------------------------

	this.memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());

	// instantiate the serializer / comparator
	final TypeSerializerFactory<T> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<T> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<T> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	this.sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	final List<MemorySegment> memory = this.memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (this.sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<T>(this.serializer, this.sortingComparator, memory);
	} else {
		this.sorter = new NormalizedKeySorter<T>(this.serializer, this.sortingComparator.duplicate(), memory);
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void prepare() throws Exception {
	if(this.taskContext.getTaskConfig().getDriverStrategy() != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + this.taskContext.getTaskConfig().getDriverStrategy() + " for " +
				"group reduce combinder.");
	}

	this.memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());

	final TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();
	this.sortingComparator = this.taskContext.getDriverComparator(0);
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	final List<MemorySegment> memory = this.memManager.allocatePages(this.taskContext.getOwningNepheleTask(),
			numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (this.sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<T>(this.serializer, this.sortingComparator, memory);
	} else {
		this.sorter = new NormalizedKeySorter<T>(this.serializer, this.sortingComparator.duplicate(), memory);
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, true);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	this.memoryManager.release(sorter.dispose());
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Override
public void prepare() throws Exception {
	final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();

	strategy = taskContext.getTaskConfig().getDriverStrategy();

	// instantiate the serializer / comparator
	final TypeSerializerFactory<T> serializerFactory = taskContext.getInputSerializer(0);
	comparator = taskContext.getDriverComparator(0);
	serializer = serializerFactory.getSerializer();
	reducer = taskContext.getStub();
	output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);

	MemoryManager memManager = taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(
		taskContext.getTaskConfig().getRelativeMemoryDriver());
	memory = memManager.allocatePages(taskContext.getContainingTask(), numMemoryPages);

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("ReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}

	switch (strategy) {
		case SORTED_PARTIAL_REDUCE:
			// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
			if (comparator.supportsSerializationWithKeyNormalization() &&
				serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
				sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
			} else {
				sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
			}
			break;
		case HASHED_PARTIAL_REDUCE:
			table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
			reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled);
			break;
		default:
			throw new Exception("Invalid strategy " + taskContext.getTaskConfig().getDriverStrategy() + " for reduce combiner.");
	}
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
private FixedLengthRecordSorter<IntPair> newSortBuffer(List<MemorySegment> memory) throws Exception {
	return new FixedLengthRecordSorter<IntPair>(this.serializer, this.comparator, memory);
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		this.memoryManager.release(sorter.dispose());
	}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	this.memoryManager.release(sorter.dispose());
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	this.memoryManager.release(sorter.dispose());
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	this.memoryManager.release(sorter.dispose());
}
 

import org.apache.flink.runtime.operators.sort.FixedLengthRecordSorter; //导入依赖的package包/类
@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current = 0;
	int last = 0;
	
	iter.next(readTarget);
	//readTarget.getFieldInto(0, last);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		
		int tmp = current;
		current = last;
		last = tmp;
	}
	
	// release the memory occupied by the buffers
	this.memoryManager.release(sorter.dispose());
}