當前位置: 首頁>>代碼示例>>Java>>正文


Java TaskConfig.getPairComparatorFactory方法代碼示例

本文整理匯總了Java中org.apache.flink.runtime.operators.util.TaskConfig.getPairComparatorFactory方法的典型用法代碼示例。如果您正苦於以下問題:Java TaskConfig.getPairComparatorFactory方法的具體用法?Java TaskConfig.getPairComparatorFactory怎麽用?Java TaskConfig.getPairComparatorFactory使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在org.apache.flink.runtime.operators.util.TaskConfig的用法示例。


在下文中一共展示了TaskConfig.getPairComparatorFactory方法的4個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: prepare

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception
{
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
		throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
	}
	
	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
				this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for CoGroup driver");
	}

	// create CoGropuTaskIterator according to provided local strategy.
	this.coGroupIterator = new SortMergeCoGroupIterator<IT1, IT2>(in1, in2,
			serializer1, groupComparator1,  serializer2, groupComparator2,
			pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	
	// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
	this.coGroupIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}
 
開發者ID:citlab,項目名稱:vs.msc.ws14,代碼行數:36,代碼來源:CoGroupDriver.java

示例2: prepare

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception
{
	final TaskConfig config = this.taskContext.getTaskConfig();
	if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
		throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
	}

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
				this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for CoGroup driver");
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}

	if (objectReuseEnabled) {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	} else {
		// create CoGroupTaskIterator according to provided local strategy.
		this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
				in1, in2,
				serializer1, groupComparator1,
				serializer2, groupComparator2,
				pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
	}
	
	// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
	this.coGroupIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
	}
}
 
開發者ID:axbaretto,項目名稱:flink,代碼行數:56,代碼來源:CoGroupDriver.java

示例3: prepare

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double driverMemFraction = config.getRelativeMemoryDriver();
	
	final DriverStrategy ls = config.getDriverStrategy();

	final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
	final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
	final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
	
	// get serializers and comparators
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
	
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for outer join driver");
	}
	
	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
	
	// create and return outer join iterator according to provided local strategy.
	if (objectReuseEnabled) {
		this.outerJoinIterator = getReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	} else {
		this.outerJoinIterator = getNonReusingOuterJoinIterator(
				ls,
				in1,
				in2,
				serializer1,
				comparator1,
				serializer2,
				comparator2,
				pairComparatorFactory,
				memoryManager,
				ioManager,
				driverMemFraction
		);
	}
	
	this.outerJoinIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
	}
}
 
開發者ID:axbaretto,項目名稱:flink,代碼行數:74,代碼來源:AbstractOuterJoinDriver.java

示例4: prepare

import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception{
	final TaskConfig config = this.taskContext.getTaskConfig();
	
	// obtain task manager's memory manager and I/O manager
	final MemoryManager memoryManager = this.taskContext.getMemoryManager();
	final IOManager ioManager = this.taskContext.getIOManager();
	
	// set up memory and I/O parameters
	final double fractionAvailableMemory = config.getRelativeMemoryDriver();
	final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
	
	// test minimum memory requirements
	final DriverStrategy ls = config.getDriverStrategy();
	
	final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
	final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
	
	// get the key positions and types
	final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
	final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
	final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
	final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
	
	final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
			this.taskContext.getUserCodeClassLoader());
	if (pairComparatorFactory == null) {
		throw new Exception("Missing pair comparator factory for Match driver");
	}

	// create and return MatchTaskIterator according to provided local strategy.
	switch (ls) {
	case MERGE:
		this.matchIterator = new MergeMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
				serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2),
				memoryManager, ioManager, numPages, this.taskContext.getOwningNepheleTask());
		break;
	case HYBRIDHASH_BUILD_FIRST:
		this.matchIterator = new BuildFirstHashMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
			serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2),
			memoryManager, ioManager, this.taskContext.getOwningNepheleTask(), fractionAvailableMemory);
		break;
	case HYBRIDHASH_BUILD_SECOND:
		this.matchIterator = new BuildSecondHashMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
				serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2),
				memoryManager, ioManager, this.taskContext.getOwningNepheleTask(), fractionAvailableMemory);
		break;
	default:
		throw new Exception("Unsupported driver strategy for Match driver: " + ls.name());
	}
	
	// open MatchTaskIterator - this triggers the sorting or hash-table building
	// and blocks until the iterator is ready
	this.matchIterator.open();
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(this.taskContext.formatLogString("Match task iterator ready."));
	}
}
 
開發者ID:citlab,項目名稱:vs.msc.ws14,代碼行數:60,代碼來源:MatchDriver.java


注:本文中的org.apache.flink.runtime.operators.util.TaskConfig.getPairComparatorFactory方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台,相關代碼片段篩選自各路編程大神貢獻的開源項目,源碼版權歸原作者所有,傳播和使用請參考對應項目的License;未經允許,請勿轉載。