Java SpillingResettableMutableObjectIterator类代码示例

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
private void runStreamedOuterFirst() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " +
				"First input is outer side, second input is inner (spilling) side."));
	}
	
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(
			in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T1 val1Copy = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();

	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	while (this.running && ((val1 = in1.next(val1Reuse)) != null)) {
		// for all values from the spilling side
		while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
			val1Copy = serializer1.copy(val1, val1Copy);
			collector.collect(crosser.cross(val1Copy, val2));
			//crosser.cross(val1Copy, val2, collector);
		}
		spillVals.reset();
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
private void runStreamedOuterSecond() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " +
				"First input is inner (spilling) side, second input is outer side."));
	}
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(
			in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();
	T2 val2Copy = serializer2.createInstance();
	
	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	while (this.running && (val2 = in2.next(val2Reuse)) != null) {
		// for all values from the spilling side
		while (this.running && (val1 = spillVals.next(val1Reuse)) != null) {
			val2Copy = serializer2.copy(val2, val2Copy);
			collector.collect(crosser.cross(val1, val2Copy));
			//crosser.cross(val1, val2Copy, collector);
		}
		spillVals.reset();
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
/**
 *
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {

	final MemoryManager memMan = getMemoryManager();
	final IOManager ioMan = getIOManager();

	this.localStrategies = new CloseableInputProvider<?>[numInputs];
	this.inputs = new MutableObjectIterator<?>[numInputs];
	this.excludeFromReset = new boolean[numInputs];
	this.inputIsCached = new boolean[numInputs];
	this.inputIsAsyncMaterialized = new boolean[numInputs];
	this.materializationMemory = new int[numInputs];

	// set up the local strategies first, such that the can work before any temp barrier is created
	for (int i = 0; i < numInputs; i++) {
		initInputLocalStrategy(i);
	}

	// we do another loop over the inputs, because we want to instantiate all
	// sorters, etc before requesting the first input (as this call may block)

	// we have two types of materialized inputs, and both are replayable (can act as a cache)
	// The first variant materializes in a different thread and hence
	// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
	// the second variant spills to the side and will not read unless the result is also consumed
	// in a pipelined fashion.
	this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
	this.tempBarriers = new TempBarrier<?>[numInputs];

	for (int i = 0; i < numInputs; i++) {
		final int memoryPages;
		final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
		final boolean cached =  this.config.isInputCached(i);

		this.inputIsAsyncMaterialized[i] = async;
		this.inputIsCached[i] = cached;

		if (async || cached) {
			memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
			if (memoryPages <= 0) {
				throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
			}
			this.materializationMemory[i] = memoryPages;
		} else {
			memoryPages = 0;
		}

		if (async) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
			barrier.startReading();
			this.tempBarriers[i] = barrier;
			this.inputs[i] = null;
		} else if (cached) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
				getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
			this.resettableInputs[i] = iter;
			this.inputs[i] = iter;
		}
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
/**
 *
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 *
 * @param numInputs
 */
protected void initLocalStrategies(int numInputs) throws Exception {

	final MemoryManager memMan = getMemoryManager();
	final IOManager ioMan = getIOManager();

	this.localStrategies = new CloseableInputProvider[numInputs];
	this.inputs = new MutableObjectIterator[numInputs];
	this.excludeFromReset = new boolean[numInputs];
	this.inputIsCached = new boolean[numInputs];
	this.inputIsAsyncMaterialized = new boolean[numInputs];
	this.materializationMemory = new int[numInputs];

	// set up the local strategies first, such that the can work before any temp barrier is created
	for (int i = 0; i < numInputs; i++) {
		initInputLocalStrategy(i);
	}

	// we do another loop over the inputs, because we want to instantiate all
	// sorters, etc before requesting the first input (as this call may block)

	// we have two types of materialized inputs, and both are replayable (can act as a cache)
	// The first variant materializes in a different thread and hence
	// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
	// the second variant spills to the side and will not read unless the result is also consumed
	// in a pipelined fashion.
	this.resettableInputs = new SpillingResettableMutableObjectIterator[numInputs];
	this.tempBarriers = new TempBarrier[numInputs];

	for (int i = 0; i < numInputs; i++) {
		final int memoryPages;
		final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
		final boolean cached =  this.config.isInputCached(i);

		this.inputIsAsyncMaterialized[i] = async;
		this.inputIsCached[i] = cached;

		if (async || cached) {
			memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
			if (memoryPages <= 0) {
				throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
			}
			this.materializationMemory[i] = memoryPages;
		} else {
			memoryPages = 0;
		}

		if (async) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
			barrier.startReading();
			this.tempBarriers[i] = barrier;
			this.inputs[i] = null;
		} else if (cached) {
			@SuppressWarnings({ "unchecked", "rawtypes" })
			SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
				getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
			this.resettableInputs[i] = iter;
			this.inputs[i] = iter;
		}
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
private void runBlockedOuterFirst() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " +
				"First input is outer (blocking) side, second input is inner (spilling) side."));
	}
		
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final BlockResettableMutableObjectIterator<T1> blockVals = 
			new BlockResettableMutableObjectIterator<T1>(this.memManager, in1, serializer1, this.memPagesForBlockSide,
						this.taskContext.getOwningNepheleTask());
	this.blockIter = blockVals;
	
	final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(
			in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();
	T2 val2Copy = serializer2.createInstance();
	
	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	do {
		// for all values from the spilling side
		while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
			// for all values in the block
			while ((val1 = blockVals.next(val1Reuse)) != null) {
				val2Copy = serializer2.copy(val2, val2Copy);
				collector.collect(crosser.cross(val1,val2Copy));
				//crosser.cross(val1, val2Copy, collector);
			}
			blockVals.reset();
		}
		spillVals.reset();
	}
	while (this.running && blockVals.nextBlock());
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
private void runBlockedOuterSecond() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " +
				"First input is inner (spilling) side, second input is outer (blocking) side."));
	}
	
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(
			in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	final BlockResettableMutableObjectIterator<T2> blockVals = 
			new BlockResettableMutableObjectIterator<T2>(this.memManager, in2, serializer2, this.memPagesForBlockSide,
					this.taskContext.getOwningNepheleTask());
	this.blockIter = blockVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T1 val1Copy = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();

	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	do {
		// for all values from the spilling side
		while (this.running && ((val1 = spillVals.next(val1Reuse)) != null)) {
			// for all values in the block
			while (this.running && ((val2 = blockVals.next(val2Reuse)) != null)) {
				val1Copy = serializer1.copy(val1, val1Copy);
				collector.collect(crosser.cross(val1Copy, val2));
				//crosser.cross(val1Copy, val2, collector);
			}
			blockVals.reset();
		}
		spillVals.reset();
	}
	while (this.running && blockVals.nextBlock());
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
/**
 * Tests the resettable iterator with too little memory, so that the data
 * has to be written to disk.
 */
@Test
public void testResettableIterator() {
	try {
		final AbstractInvokable memOwner = new DummyInvokable();

		// create the resettable Iterator
		SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(
			this.reader, this.serializer, this.memman, this.ioman, 2, memOwner);

		// open the iterator
		iterator.open();
		
		// now test walking through the iterator
		int count = 0;
		Record target = new Record();
		while ((target = iterator.next(target)) != null) {
			Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
				target.getField(0, IntValue.class).getValue());
		}
		Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
		// test resetting the iterator a few times
		for (int j = 0; j < 10; ++j) {
			count = 0;
			iterator.reset();
			target = new Record();
			// now we should get the same results
			while ((target = iterator.next(target)) != null) {
				Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
					+ " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
			}
			Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
				count);
		}
		// close the iterator
		iterator.close();
	} catch (Exception ex)  {
		ex.printStackTrace();
		Assert.fail("Test encountered an exception.");
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入依赖的package包/类
/**
 * Tests the resettable iterator with enough memory so that all data is kept locally in memory.
 */
@Test
public void testResettableIteratorInMemory() {
	try {
		final AbstractInvokable memOwner = new DummyInvokable();

		// create the resettable Iterator
		SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(
			this.reader, this.serializer, this.memman, this.ioman, 20, memOwner);
		
		// open the iterator
		iterator.open();

		// now test walking through the iterator
		int count = 0;
		Record target = new Record();
		while ((target = iterator.next(target)) != null) {
			Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
				target.getField(0, IntValue.class).getValue());
		}
		Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
		// test resetting the iterator a few times
		for (int j = 0; j < 10; ++j) {
			count = 0;
			iterator.reset();
			target = new Record();
			// now we should get the same results
			while ((target = iterator.next(target)) != null) {
				Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
					+ " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
			}
			Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
				count);
		}
		// close the iterator
		iterator.close();
	} catch (Exception ex)  {
		ex.printStackTrace();
		Assert.fail("Test encountered an exception.");
	}
}