Java SpillingResettableMutableObjectIterator.next方法代码示例

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
private void runStreamedOuterFirst() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " +
				"First input is outer side, second input is inner (spilling) side."));
	}
	
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(
			in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T1 val1Copy = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();

	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	while (this.running && ((val1 = in1.next(val1Reuse)) != null)) {
		// for all values from the spilling side
		while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
			val1Copy = serializer1.copy(val1, val1Copy);
			collector.collect(crosser.cross(val1Copy, val2));
			//crosser.cross(val1Copy, val2, collector);
		}
		spillVals.reset();
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
private void runStreamedOuterSecond() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " +
				"First input is inner (spilling) side, second input is outer side."));
	}
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(
			in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();
	T2 val2Copy = serializer2.createInstance();
	
	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	while (this.running && (val2 = in2.next(val2Reuse)) != null) {
		// for all values from the spilling side
		while (this.running && (val1 = spillVals.next(val1Reuse)) != null) {
			val2Copy = serializer2.copy(val2, val2Copy);
			collector.collect(crosser.cross(val1, val2Copy));
			//crosser.cross(val1, val2Copy, collector);
		}
		spillVals.reset();
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
private void runBlockedOuterFirst() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " +
				"First input is outer (blocking) side, second input is inner (spilling) side."));
	}
		
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final BlockResettableMutableObjectIterator<T1> blockVals = 
			new BlockResettableMutableObjectIterator<T1>(this.memManager, in1, serializer1, this.memPagesForBlockSide,
						this.taskContext.getOwningNepheleTask());
	this.blockIter = blockVals;
	
	final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(
			in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();
	T2 val2Copy = serializer2.createInstance();
	
	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	do {
		// for all values from the spilling side
		while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
			// for all values in the block
			while ((val1 = blockVals.next(val1Reuse)) != null) {
				val2Copy = serializer2.copy(val2, val2Copy);
				collector.collect(crosser.cross(val1,val2Copy));
				//crosser.cross(val1, val2Copy, collector);
			}
			blockVals.reset();
		}
		spillVals.reset();
	}
	while (this.running && blockVals.nextBlock());
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
private void runBlockedOuterSecond() throws Exception {
	if (LOG.isDebugEnabled())  {
		LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " +
				"First input is inner (spilling) side, second input is outer (blocking) side."));
	}
	
	final MutableObjectIterator<T1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<T2> in2 = this.taskContext.getInput(1);
	
	final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
	final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
	
	final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(
			in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide,
			this.taskContext.getOwningNepheleTask());
	this.spillIter = spillVals;
	
	final BlockResettableMutableObjectIterator<T2> blockVals = 
			new BlockResettableMutableObjectIterator<T2>(this.memManager, in2, serializer2, this.memPagesForBlockSide,
					this.taskContext.getOwningNepheleTask());
	this.blockIter = blockVals;
	
	T1 val1;
	final T1 val1Reuse = serializer1.createInstance();
	T1 val1Copy = serializer1.createInstance();
	T2 val2;
	final T2 val2Reuse = serializer2.createInstance();

	final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	
	// for all blocks
	do {
		// for all values from the spilling side
		while (this.running && ((val1 = spillVals.next(val1Reuse)) != null)) {
			// for all values in the block
			while (this.running && ((val2 = blockVals.next(val2Reuse)) != null)) {
				val1Copy = serializer1.copy(val1, val1Copy);
				collector.collect(crosser.cross(val1Copy, val2));
				//crosser.cross(val1Copy, val2, collector);
			}
			blockVals.reset();
		}
		spillVals.reset();
	}
	while (this.running && blockVals.nextBlock());
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
/**
 * Tests the resettable iterator with too little memory, so that the data
 * has to be written to disk.
 */
@Test
public void testResettableIterator() {
	try {
		final AbstractInvokable memOwner = new DummyInvokable();

		// create the resettable Iterator
		SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(
			this.reader, this.serializer, this.memman, this.ioman, 2, memOwner);

		// open the iterator
		iterator.open();
		
		// now test walking through the iterator
		int count = 0;
		Record target = new Record();
		while ((target = iterator.next(target)) != null) {
			Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
				target.getField(0, IntValue.class).getValue());
		}
		Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
		// test resetting the iterator a few times
		for (int j = 0; j < 10; ++j) {
			count = 0;
			iterator.reset();
			target = new Record();
			// now we should get the same results
			while ((target = iterator.next(target)) != null) {
				Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
					+ " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
			}
			Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
				count);
		}
		// close the iterator
		iterator.close();
	} catch (Exception ex)  {
		ex.printStackTrace();
		Assert.fail("Test encountered an exception.");
	}
}
 

import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator; //导入方法依赖的package包/类
/**
 * Tests the resettable iterator with enough memory so that all data is kept locally in memory.
 */
@Test
public void testResettableIteratorInMemory() {
	try {
		final AbstractInvokable memOwner = new DummyInvokable();

		// create the resettable Iterator
		SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(
			this.reader, this.serializer, this.memman, this.ioman, 20, memOwner);
		
		// open the iterator
		iterator.open();

		// now test walking through the iterator
		int count = 0;
		Record target = new Record();
		while ((target = iterator.next(target)) != null) {
			Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++,
				target.getField(0, IntValue.class).getValue());
		}
		Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
		// test resetting the iterator a few times
		for (int j = 0; j < 10; ++j) {
			count = 0;
			iterator.reset();
			target = new Record();
			// now we should get the same results
			while ((target = iterator.next(target)) != null) {
				Assert.assertEquals("After reset nr. " + j + 1 + " element " + count
					+ " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
			}
			Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS,
				count);
		}
		// close the iterator
		iterator.close();
	} catch (Exception ex)  {
		ex.printStackTrace();
		Assert.fail("Test encountered an exception.");
	}
}