当前位置: 首页>>代码示例>>Java>>正文


Java TaskContext类代码示例

本文整理汇总了Java中org.apache.spark.TaskContext的典型用法代码示例。如果您正苦于以下问题:Java TaskContext类的具体用法?Java TaskContext怎么用?Java TaskContext使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。


TaskContext类属于org.apache.spark包,在下文中一共展示了TaskContext类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: call

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override 
public Iterator<Tuple2<Long, String>> call(Task arg0)
	throws Exception 
{
	//lazy parworker initialization
	if( !_initialized )
		configureWorker(TaskContext.get().taskAttemptId());
	
	//execute a single task
	long numIter = getExecutedIterations();
	super.executeTask( arg0 );
	
	//maintain accumulators
	_aTasks.add( 1 );
	_aIters.add( (int)(getExecutedIterations()-numIter) );
	
	//write output if required (matrix indexed write) 
	//note: this copy is necessary for environments without spark libraries
	ArrayList<Tuple2<Long,String>> ret = new ArrayList<>();
	ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars );
	for( String val : tmp )
		ret.add(new Tuple2<>(_workerID, val));
	
	return ret.iterator();
}
 
开发者ID:apache,项目名称:systemml,代码行数:26,代码来源:RemoteParForSparkWorker.java

示例2: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<double[]> compute(final Partition partition, final TaskContext context) {
    ProgrammingError.throwIfNull(partition, context);
    if (partition instanceof Partition2D) {
        return this.compute((Partition2D) partition, context);
    } else {
        throw new IllegalArgumentException();
    }
}
 
开发者ID:optimatika,项目名称:ojAlgo-extensions,代码行数:10,代码来源:PrimitiveBlockMatrixRDD.java

示例3: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<MatrixStore<N>> compute(final Partition partition, final TaskContext context) {
    ProgrammingError.throwIfNull(partition, context);
    if (partition instanceof Partition2D) {
        return this.compute((Partition2D) partition, context);
    } else {
        throw new IllegalArgumentException();
    }
}
 
开发者ID:optimatika,项目名称:ojAlgo-extensions,代码行数:10,代码来源:OtherBlockMatrixRDD.java

示例4: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<TReturn> compute(Partition split, TaskContext context) {
    String regionEdgesFamilyPath = this.regionsPaths.get(split.index());
    log.info("Running Mizo on region #{} located at: {}", split.index(), regionEdgesFamilyPath);

    return createRegionIterator(createRegionRelationsIterator(regionEdgesFamilyPath));
}
 
开发者ID:imri,项目名称:mizo,代码行数:8,代码来源:MizoRDD.java

示例5: call

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<Tuple2<Integer, String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>> iter) throws Exception
{
  List<Tuple2<Integer,String>> keyFileList = new ArrayList<>();

  FileSystem fs = FileSystem.get(new Configuration());

  // Form the filename for the exp table portion that corresponds to this partition
  int taskId = TaskContext.getPartitionId();
  logger.info("taskId = " + taskId);

  String fileName = expOutDir + "/exp-" + String.format("%05d", taskId);
  logger.info("fileName = " + fileName);

  // Iterate over the elements of the partition
  BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(fileName), true)));
  while (iter.hasNext())
  {
    // <queryHash, <<power>,<element^power mod N^2>>
    Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>> expTuple = iter.next();
    int queryHash = expTuple._1;

    // Record the queryHash -> fileName
    keyFileList.add(new Tuple2<>(queryHash, fileName));

    // Write the partition elements to the corresponding exp table file
    // each line: queryHash,<power>-<element^power mod N^2>
    for (Tuple2<Integer,BigInteger> modPow : expTuple._2)
    {
      String lineOut = queryHash + "," + modPow._1 + "-" + modPow._2;
      bw.write(lineOut);
      bw.newLine();
    }
  }
  bw.close();

  return keyFileList.iterator();
}
 
开发者ID:apache,项目名称:incubator-pirk,代码行数:39,代码来源:ExpKeyFilenameMap.java

示例6: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<WindowedValue<T>> compute(final Partition split,
                                                           final TaskContext context) {
  final MetricsContainer metricsContainer = metricsAccum.localValue().getContainer(stepName);

  @SuppressWarnings("unchecked")
  final BoundedSource.BoundedReader<T> reader = createReader((SourcePartition<T>) split);

  final Iterator<WindowedValue<T>> readerIterator =
      new ReaderToIteratorAdapter<>(metricsContainer, reader);

  return new InterruptibleIterator<>(context, JavaConversions.asScalaIterator(readerIterator));
}
 
开发者ID:apache,项目名称:beam,代码行数:14,代码来源:SourceRDD.java

示例7: addOnCompletition

import org.apache.spark.TaskContext; //导入依赖的package包/类
static void addOnCompletition(TaskContext taskContext, final Function0<?> function) {
    taskContext.addOnCompleteCallback(new AbstractFunction0() {
        @Override
        public BoxedUnit apply() {
            function.apply();
            return BoxedUnit.UNIT;
        }
    });

}
 
开发者ID:xushjie1987,项目名称:es-hadoop-v2.2.0,代码行数:11,代码来源:CompatUtils.java

示例8: buildWriter

import org.apache.spark.TaskContext; //导入依赖的package包/类
private ParquetWriter<Element> buildWriter(final String group, final String column, final boolean isEntity, final int splitNumber) throws IOException {
    LOGGER.debug("Creating a new writer for group: {}", group + " with file number " + splitNumber);
    final Path filePath = new Path(ParquetStore.getGroupDirectory(group, column,
            tempFilesDir) + "/raw/split" + splitNumber + "/part-" + TaskContext.getPartitionId() + ".parquet");

    return new ParquetElementWriter.Builder(filePath)
            .isEntity(isEntity)
            .withType(schemaUtils.getParquetSchema(group))
            .usingConverter(schemaUtils.getConverter(group))
            .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
            .withSparkSchema(schemaUtils.getSparkSchema(group))
            .build();
}
 
开发者ID:gchq,项目名称:Gaffer,代码行数:14,代码来源:WriteUnsortedData.java

示例9: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<Map.Entry<Key, Value>> compute(final Partition split, final TaskContext context) {
    final ByteArrayInputStream bais = new ByteArrayInputStream(serialisedConfiguration);
    final Configuration configuration = new Configuration();
    try {
        configuration.readFields(new DataInputStream(bais));
        bais.close();
    } catch (final IOException e) {
        throw new RuntimeException("IOException deserialising Configuration from byte array", e);
    }
    return new InterruptibleIterator<>(context,
            JavaConversions.asScalaIterator(new RFileReaderIterator(split, context, configuration, auths)));
}
 
开发者ID:gchq,项目名称:Gaffer,代码行数:14,代码来源:RFileReaderRDD.java

示例10: RFileReaderIterator

import org.apache.spark.TaskContext; //导入依赖的package包/类
public RFileReaderIterator(final Partition partition,
                           final TaskContext taskContext,
                           final Configuration configuration,
                           final Set<String> auths) {
    this.partition = partition;
    this.taskContext = taskContext;
    this.configuration = configuration;
    this.auths = auths;
    try {
        init();
    } catch (final IOException e) {
        throw new RuntimeException("IOException initialising RFileReaderIterator", e);
    }
}
 
开发者ID:gchq,项目名称:Gaffer,代码行数:15,代码来源:RFileReaderIterator.java

示例11: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<IndexedContent> compute(Partition split, TaskContext context) {
    try {
        Path path = new Path(partitions[split.index()].getPath());
        java.util.Iterator<Content> iterator = new ContentIterator(path, NutchConfiguration.create(), contentTypeFilter);
        return new IndexedContentIterator(iterator);
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new RuntimeException(e);
    }
}
 
开发者ID:thammegowda,项目名称:autoextractor,代码行数:12,代码来源:IndexedNutchContentRDD.java

示例12: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<Content> compute(Partition split, TaskContext context) {
    try {
        Path path = new Path(partitions[split.index()].getPath());
        return new ContentIterator(path, NutchConfiguration.create(), contentTypeFilter);
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new RuntimeException(e);
    }
}
 
开发者ID:thammegowda,项目名称:autoextractor,代码行数:11,代码来源:NutchContentRDD.java

示例13: log

import org.apache.spark.TaskContext; //导入依赖的package包/类
private void log(String message, Object...params) {
    if (Level.INFO.isGreaterOrEqual(log.getEffectiveLevel())) {
        final int partitionId = TaskContext.getPartitionId();
        final long threadId = Thread.currentThread().getId();
        log.info("[" + threadId + ", PID=" + partitionId + "] " + String.format(message, params));
    }
}
 
开发者ID:DataSystemsLab,项目名称:GeoSpark,代码行数:8,代码来源:DynamicIndexLookupJudgement.java

示例14: initPartition

import org.apache.spark.TaskContext; //导入依赖的package包/类
/**
 * Looks up the extent of the current partition. If found, `match` method will
 * activate the logic to avoid emitting duplicate join results from multiple partitions.
 *
 * Must be called before processing a partition. Must be called from the
 * same instance that will be used to process the partition.
 */
protected void initPartition() {
    if (dedupParams == null) {
        return;
    }

    final int partitionId = TaskContext.getPartitionId();

    final List<Envelope> partitionExtents = dedupParams.getPartitionExtents();
    if (partitionId < partitionExtents.size()) {
        extent = new HalfOpenRectangle(partitionExtents.get(partitionId));
    } else {
        log.warn("Didn't find partition extent for this partition: " + partitionId);
    }
}
 
开发者ID:DataSystemsLab,项目名称:GeoSpark,代码行数:22,代码来源:JudgementBase.java

示例15: compute

import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<T> compute(Partition split, TaskContext context) {

    initExtractorClient();

    extractorClient.initIterator(split, config.getValue());

    context.addTaskCompletionListener(new AbstractFunction1<TaskContext, BoxedUnit>() {

        @Override
        public BoxedUnit apply(TaskContext v1) {
            extractorClient.close();
            return null;
        }
    });

    java.util.Iterator<T> iterator = new java.util.Iterator<T>() {

        @Override
        public boolean hasNext() {
            return extractorClient.hasNext();
        }

        @Override
        public T next() {
            return extractorClient.next();
        }

        @Override
        public void remove() {
            throw new DeepIOException(
                    "Method not implemented (and won't be implemented anytime soon!!!)");
        }
    };

    return new InterruptibleIterator<>(context, asScalaIterator(iterator));

}
 
开发者ID:Stratio,项目名称:deep-spark,代码行数:39,代码来源:DeepRDD.java


注:本文中的org.apache.spark.TaskContext类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。