当前位置: 首页>>代码示例>>Java>>正文


Java BlockChannelReader类代码示例

本文整理汇总了Java中org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader的典型用法代码示例。如果您正苦于以下问题:Java BlockChannelReader类的具体用法?Java BlockChannelReader怎么用?Java BlockChannelReader使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。


BlockChannelReader类属于org.apache.flink.runtime.io.disk.iomanager包,在下文中一共展示了BlockChannelReader类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: ReadEnd

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
private ReadEnd(MemorySegment firstMemSegment, LinkedBlockingQueue<MemorySegment> emptyBufferTarget,
								Deque<MemorySegment> fullBufferSource, BlockChannelReader<MemorySegment> spilledBufferSource,
								List<MemorySegment> emptyBuffers, int numBuffersSpilled)
	throws IOException {
	super(firstMemSegment, firstMemSegment.getInt(0), HEADER_LENGTH);

	this.emptyBufferTarget = emptyBufferTarget;
	this.fullBufferSource = fullBufferSource;

	this.spilledBufferSource = spilledBufferSource;

	requestsRemaining = numBuffersSpilled;
	this.spilledBuffersRemaining = numBuffersSpilled;

	// send the first requests
	while (requestsRemaining > 0 && emptyBuffers.size() > 0) {
		this.spilledBufferSource.readBlock(emptyBuffers.remove(emptyBuffers.size() - 1));
		requestsRemaining--;
	}
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:21,代码来源:SerializedUpdateBuffer.java

示例2: ReadEnd

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
private ReadEnd(MemorySegment firstMemSegment, LinkedBlockingQueue<MemorySegment> emptyBufferTarget,
		ArrayDeque<MemorySegment> fullBufferSource, BlockChannelReader spilledBufferSource,
		ArrayList<MemorySegment> emptyBuffers, int segmentSize, int numBuffersSpilled)
	throws IOException {
	super(firstMemSegment, firstMemSegment.getInt(0), HEADER_LENGTH);

	this.emptyBufferTarget = emptyBufferTarget;
	this.fullBufferSource = fullBufferSource;

	this.spilledBufferSource = spilledBufferSource;

	requestsRemaining = numBuffersSpilled;
	this.spilledBuffersRemaining = numBuffersSpilled;

	// send the first requests
	while (requestsRemaining > 0 && emptyBuffers.size() > 0) {
		this.spilledBufferSource.readBlock(emptyBuffers.remove(emptyBuffers.size() - 1));
		requestsRemaining--;
	}
}
 
开发者ID:citlab,项目名称:vs.msc.ws14,代码行数:21,代码来源:SerializedUpdateBuffer.java

示例3: getMergingIterator

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:45,代码来源:UnilateralSortMerger.java

示例4: ChannelReaderInputViewIterator

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
public ChannelReaderInputViewIterator(BlockChannelReader<MemorySegment> reader, LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this.accessors = accessors;
	this.freeMemTarget = freeMemTarget;
	this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:9,代码来源:ChannelReaderInputViewIterator.java

示例5: FileChannelInputView

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
public FileChannelInputView(BlockChannelReader<MemorySegment> reader, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(reader);
	checkNotNull(memManager);
	checkNotNull(memory);
	checkArgument(!reader.isClosed());
	checkArgument(memory.size() > 0);
	
	this.reader = reader;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	
	try {
		final long channelLength = reader.getSize();
		final int segmentSize = memManager.getPageSize();
		
		this.numBlocksRemaining = MathUtils.checkedDownCast(channelLength / segmentSize);
		if (channelLength % segmentSize != 0) {
			this.numBlocksRemaining++;
		}
		
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:37,代码来源:FileChannelInputView.java

示例6: testCloseAndDeleteInputView

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testCloseAndDeleteInputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		
		// add some test data
		try (FileWriter wrt = new FileWriter(channel.getPath())) {
			wrt.write("test data");
		}
		
		BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
		FileChannelInputView in = new FileChannelInputView(reader, memMan, memory, 9);
		
		// read just something
		in.readInt();
		
		// close for the first time, make sure all memory returns
		in.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		in.close();
		
		// delete, make sure file is removed
		in.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:41,代码来源:FileChannelStreamsTest.java

示例7: testWriteAndReadLongRecords

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testWriteAndReadLongRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		final int k1 = rec.f0;
		final String v1 = rec.f1;
		final int k2 = readRec.f0;
		final String v2 = readRec.f1;
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:42,代码来源:ChannelViewsTest.java

示例8: getMergingIterator

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<BlockChannelAccess<?, ?>> readerList)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size());
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader reader = segsForChannel.size() >= 4 ? 
			this.ioManager.createBlockChannelReader(channel.getChannel(), segsForChannel.size() / 2) :
			this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}

	return new MergeIterator<E>(iterators, this.serializer, this.comparator);
}
 
开发者ID:citlab,项目名称:vs.msc.ws14,代码行数:43,代码来源:UnilateralSortMerger.java

示例9: ChannelReaderInputViewIterator

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
public ChannelReaderInputViewIterator(BlockChannelReader reader, LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this.accessors = accessors;
	this.freeMemTarget = freeMemTarget;
	this.inView = new ChannelReaderInputView(reader, segments, numBlocks, false);
}
 
开发者ID:citlab,项目名称:vs.msc.ws14,代码行数:9,代码来源:ChannelReaderInputViewIterator.java

示例10: testWriteAndReadLongRecords

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testWriteAndReadLongRecords() throws Exception
{
	final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final Channel.ID channel = this.ioManager.createChannel();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Record rec = new Record();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		rec.write(outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Record readRec = new Record();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		readRec.read(inView);
		final Key k1 = rec.getField(0, Key.class);
		final Value v1 = rec.getField(1, Value.class);
		final Key k2 = readRec.getField(0, Key.class);
		final Value v2 = readRec.getField(1, Value.class);
		Assert.assertTrue("The re-generated and the read record do not match.", k1.equals(k2) && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 
开发者ID:citlab,项目名称:vs.msc.ws14,代码行数:41,代码来源:ChannelViewsTest.java

示例11: flip

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
public DataInputView flip() throws IOException {
	// check whether this is the first flip and we need to add the current segment to the full ones
	if (getCurrentSegment() != null) {
		// first flip
		if (this.writer == null) {
			// in memory
			this.fullSegments.add(getCurrentSegment());
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.inMemInView = new RandomAccessInputView(this.fullSegments, this.segmentSize, this.numBytesInLastSegment);
		} else {
			// external: write the last segment and collect the memory back
			this.writer.writeBlock(this.getCurrentSegment());
			this.numMemorySegmentsInWriter++;
			
			this.numBytesInLastSegment = getCurrentPositionInSegment();
			this.blockCount++;
			this.writer.close();
			for (int i = this.numMemorySegmentsInWriter; i > 0; i--) {
				this.fullSegments.add(this.writer.getNextReturnedBlock());
			}
			this.numMemorySegmentsInWriter = 0;
		}
		
		// make sure we cannot write more
		clear();
	}
	
	if (this.writer == null) {
		// in memory
		this.inMemInView.setReadPosition(0);
		return this.inMemInView;
	} else {
		// recollect memory from a previous view
		if (this.externalInView != null) {
			this.externalInView.close();
		}
		
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(this.writer.getChannelID());
		this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false);
		return this.externalInView;
	}
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:43,代码来源:SpillingBuffer.java

示例12: testFlushFullMemoryPage

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testFlushFullMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 0, NUM_RECORDS);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =0;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:49,代码来源:FixedLengthRecordSorterTest.java

示例13: testFlushPartialMemoryPage

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testFlushPartialMemoryPage() throws Exception {
	// Insert IntPair which would fill 2 memory pages.
	final int NUM_RECORDS = 2 * MEMORY_PAGE_SIZE / 8;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), 3);

	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	UniformIntPairGenerator generator = new UniformIntPairGenerator(Integer.MAX_VALUE, 1, false);

	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);

	FileIOChannel.ID channelID = this.ioManager.createChannelEnumerator().next();
	BlockChannelWriter<MemorySegment> blockChannelWriter = this.ioManager.createBlockChannelWriter(channelID);
	final List<MemorySegment> writeBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelWriterOutputView outputView = new ChannelWriterOutputView(blockChannelWriter, writeBuffer, writeBuffer.get(0).size());

	sorter.writeToOutput(outputView, 1, NUM_RECORDS - 1);

	this.memoryManager.release(outputView.close());

	BlockChannelReader<MemorySegment> blockChannelReader = this.ioManager.createBlockChannelReader(channelID);
	final List<MemorySegment> readBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputView readerInputView = new ChannelReaderInputView(blockChannelReader, readBuffer, false);
	final List<MemorySegment> dataBuffer = this.memoryManager.allocatePages(new DummyInvokable(), 3);
	ChannelReaderInputViewIterator<IntPair> iterator = new ChannelReaderInputViewIterator(readerInputView, dataBuffer, this.serializer);

	record = iterator.next(record);
	int i =1;
	while (record != null) {
		Assert.assertEquals(i, record.getKey());
		record = iterator.next(record);
		i++;
	}

	Assert.assertEquals(NUM_RECORDS, i);

	this.memoryManager.release(dataBuffer);
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:49,代码来源:FixedLengthRecordSorterTest.java

示例14: testWriteReadSmallRecords

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testWriteReadSmallRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:44,代码来源:ChannelViewsTest.java

示例15: testReadTooMany

import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader; //导入依赖的package包/类
@Test
public void testReadTooMany() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());

	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();

	// read and re-generate all records and compare them
	try {
		final Tuple2<Integer, String> readRec = new Tuple2<>();
		for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
			generator.next(rec);
			serializer.deserialize(readRec, inView);
			final int k1 = rec.f0;
			final String v1 = rec.f1;
			final int k2 = readRec.f0;
			final String v2 = readRec.f1;
			Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
		}
		Assert.fail("Expected an EOFException which did not occur.");
	}
	catch (EOFException eofex) {
		// expected
	}
	catch (Throwable t) {
		// unexpected
		Assert.fail("Unexpected Exception: " + t.getMessage());
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 
开发者ID:axbaretto,项目名称:flink,代码行数:52,代码来源:ChannelViewsTest.java


注:本文中的org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。