本文整理匯總了Java中org.apache.flink.runtime.operators.util.TaskConfig.getDriverStrategy方法的典型用法代碼示例。如果您正苦於以下問題:Java TaskConfig.getDriverStrategy方法的具體用法?Java TaskConfig.getDriverStrategy怎麽用?Java TaskConfig.getDriverStrategy使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.flink.runtime.operators.util.TaskConfig
的用法示例。
在下文中一共展示了TaskConfig.getDriverStrategy方法的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Java代碼示例。
示例1: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
}
TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
this.serializer = serializerFactory.getSerializer();
this.input = this.taskContext.getInput(0);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("AllReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
示例2: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.CO_GROUP_RAW) {
throw new Exception("Unrecognized driver strategy for CoGoup Python driver: " + config.getDriverStrategy().name());
}
final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
IT1 reuse1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer().createInstance();
IT2 reuse2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer().createInstance();
this.coGroupIterator1 = new SimpleIterable<IT1>(reuse1, in1);
this.coGroupIterator2 = new SimpleIterable<IT2>(reuse2, in2);
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
}
}
示例3: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
}
this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
this.comparator = this.taskContext.getDriverComparator(0);
this.input = this.taskContext.getInput(0);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("ReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
示例4: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
}
final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
this.comparator = this.taskContext.getDriverComparator(0);
this.input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("GroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
示例5: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
this.strategy = config.getDriverStrategy();
if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
if (!(this.taskContext.getStub() instanceof FlatCombineFunction)) {
throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + FlatCombineFunction.class.getName());
}
}
else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + config.getDriverStrategy().name());
}
this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
this.input = this.taskContext.getInput(0);
}
示例6: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
}
this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
this.comparator = this.taskContext.getDriverComparator(0);
this.input = this.taskContext.getInput(0);
this.mutableObjectMode = config.getMutableObjectMode();
if (LOG.isDebugEnabled()) {
LOG.debug("GroupReduceDriver uses " + (this.mutableObjectMode ? "MUTABLE" : "IMMUTABLE") + " object mode.");
}
}
示例7: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
this.strategy = config.getDriverStrategy();
switch (this.strategy) {
case ALL_GROUP_REDUCE_COMBINE:
if (!(this.taskContext.getStub() instanceof GroupCombineFunction)) {
throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GroupCombineFunction.class.getName());
}
case ALL_GROUP_REDUCE:
case ALL_GROUP_COMBINE:
break;
default:
throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + this.strategy.name());
}
this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
this.input = this.taskContext.getInput(0);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("AllGroupReduceDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
示例8: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
}
TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
this.serializer = serializerFactory.getSerializer();
this.input = this.taskContext.getInput(0);
}
示例9: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception
{
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
}
final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
// get the key positions and types
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for CoGroup driver");
}
// create CoGropuTaskIterator according to provided local strategy.
this.coGroupIterator = new SortMergeCoGroupIterator<IT1, IT2>(in1, in2,
serializer1, groupComparator1, serializer2, groupComparator2,
pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
this.coGroupIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
}
}
示例10: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
}
this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
this.comparator = this.taskContext.getDriverComparator(0);
this.input = this.taskContext.getInput(0);
}
示例11: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception
{
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
}
final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
// get the key positions and types
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for CoGroup driver");
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
if (objectReuseEnabled) {
// create CoGroupTaskIterator according to provided local strategy.
this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
in1, in2,
serializer1, groupComparator1,
serializer2, groupComparator2,
pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
} else {
// create CoGroupTaskIterator according to provided local strategy.
this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
in1, in2,
serializer1, groupComparator1,
serializer2, groupComparator2,
pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
}
// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
this.coGroupIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
}
}
示例12: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
final DriverStrategy ls = config.getDriverStrategy();
switch (ls)
{
case NESTEDLOOP_BLOCKED_OUTER_FIRST:
this.blocked = true;
this.firstIsOuter = true;
break;
case NESTEDLOOP_BLOCKED_OUTER_SECOND:
this.blocked = true;
this.firstIsOuter = false;
break;
case NESTEDLOOP_STREAMED_OUTER_FIRST:
this.blocked = false;
this.firstIsOuter = true;
break;
case NESTEDLOOP_STREAMED_OUTER_SECOND:
this.blocked = false;
this.firstIsOuter = false;
break;
default:
throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
}
this.memManager = this.taskContext.getMemoryManager();
final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
if (numPages < 2) {
throw new RuntimeException( "The Cross task was initialized with too little memory. " +
"Cross requires at least 2 memory pages.");
}
// divide memory between spilling and blocking side
if (ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST || ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND) {
this.memPagesForSpillingSide = numPages;
this.memPagesForBlockSide = 0;
} else {
if (numPages > 32) {
this.memPagesForSpillingSide = 2;
} else {
this.memPagesForSpillingSide = 1;
}
this.memPagesForBlockSide = numPages - this.memPagesForSpillingSide;
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("CrossDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
示例13: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
// obtain task manager's memory manager and I/O manager
final MemoryManager memoryManager = this.taskContext.getMemoryManager();
final IOManager ioManager = this.taskContext.getIOManager();
// set up memory and I/O parameters
final double driverMemFraction = config.getRelativeMemoryDriver();
final DriverStrategy ls = config.getDriverStrategy();
final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
// get serializers and comparators
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for outer join driver");
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
// create and return outer join iterator according to provided local strategy.
if (objectReuseEnabled) {
this.outerJoinIterator = getReusingOuterJoinIterator(
ls,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory,
memoryManager,
ioManager,
driverMemFraction
);
} else {
this.outerJoinIterator = getNonReusingOuterJoinIterator(
ls,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory,
memoryManager,
ioManager,
driverMemFraction
);
}
this.outerJoinIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
}
}
示例14: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
final DriverStrategy ls = config.getDriverStrategy();
switch (ls)
{
case NESTEDLOOP_BLOCKED_OUTER_FIRST:
this.blocked = true;
this.firstIsOuter = true;
break;
case NESTEDLOOP_BLOCKED_OUTER_SECOND:
this.blocked = true;
this.firstIsOuter = false;
break;
case NESTEDLOOP_STREAMED_OUTER_FIRST:
this.blocked = false;
this.firstIsOuter = true;
break;
case NESTEDLOOP_STREAMED_OUTER_SECOND:
this.blocked = false;
this.firstIsOuter = false;
break;
default:
throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
}
this.memManager = this.taskContext.getMemoryManager();
final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
if (numPages < 2) {
throw new RuntimeException( "The Cross task was initialized with too little memory. " +
"Cross requires at least 2 memory pages.");
}
// divide memory between spilling and blocking side
if (ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST || ls == DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND) {
this.memPagesForSpillingSide = numPages;
this.memPagesForBlockSide = 0;
} else {
if (numPages > 32) {
this.memPagesForSpillingSide = 2;
} else {
this.memPagesForSpillingSide = 1;
}
this.memPagesForBlockSide = numPages - this.memPagesForSpillingSide;
}
}
示例15: prepare
import org.apache.flink.runtime.operators.util.TaskConfig; //導入方法依賴的package包/類
@Override
public void prepare() throws Exception{
final TaskConfig config = this.taskContext.getTaskConfig();
// obtain task manager's memory manager and I/O manager
final MemoryManager memoryManager = this.taskContext.getMemoryManager();
final IOManager ioManager = this.taskContext.getIOManager();
// set up memory and I/O parameters
final double fractionAvailableMemory = config.getRelativeMemoryDriver();
final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
// test minimum memory requirements
final DriverStrategy ls = config.getDriverStrategy();
final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
// get the key positions and types
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for Match driver");
}
// create and return MatchTaskIterator according to provided local strategy.
switch (ls) {
case MERGE:
this.matchIterator = new MergeMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager, numPages, this.taskContext.getOwningNepheleTask());
break;
case HYBRIDHASH_BUILD_FIRST:
this.matchIterator = new BuildFirstHashMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager, this.taskContext.getOwningNepheleTask(), fractionAvailableMemory);
break;
case HYBRIDHASH_BUILD_SECOND:
this.matchIterator = new BuildSecondHashMatchIterator<IT1, IT2, OT>(in1, in2, serializer1, comparator1,
serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager, this.taskContext.getOwningNepheleTask(), fractionAvailableMemory);
break;
default:
throw new Exception("Unsupported driver strategy for Match driver: " + ls.name());
}
// open MatchTaskIterator - this triggers the sorting or hash-table building
// and blocks until the iterator is ready
this.matchIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("Match task iterator ready."));
}
}