本文整理汇总了Java中org.apache.spark.TaskContext类的典型用法代码示例。如果您正苦于以下问题:Java TaskContext类的具体用法?Java TaskContext怎么用?Java TaskContext使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
TaskContext类属于org.apache.spark包,在下文中一共展示了TaskContext类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: call
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<Tuple2<Long, String>> call(Task arg0)
throws Exception
{
//lazy parworker initialization
if( !_initialized )
configureWorker(TaskContext.get().taskAttemptId());
//execute a single task
long numIter = getExecutedIterations();
super.executeTask( arg0 );
//maintain accumulators
_aTasks.add( 1 );
_aIters.add( (int)(getExecutedIterations()-numIter) );
//write output if required (matrix indexed write)
//note: this copy is necessary for environments without spark libraries
ArrayList<Tuple2<Long,String>> ret = new ArrayList<>();
ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars );
for( String val : tmp )
ret.add(new Tuple2<>(_workerID, val));
return ret.iterator();
}
示例2: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<double[]> compute(final Partition partition, final TaskContext context) {
ProgrammingError.throwIfNull(partition, context);
if (partition instanceof Partition2D) {
return this.compute((Partition2D) partition, context);
} else {
throw new IllegalArgumentException();
}
}
示例3: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<MatrixStore<N>> compute(final Partition partition, final TaskContext context) {
ProgrammingError.throwIfNull(partition, context);
if (partition instanceof Partition2D) {
return this.compute((Partition2D) partition, context);
} else {
throw new IllegalArgumentException();
}
}
示例4: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<TReturn> compute(Partition split, TaskContext context) {
String regionEdgesFamilyPath = this.regionsPaths.get(split.index());
log.info("Running Mizo on region #{} located at: {}", split.index(), regionEdgesFamilyPath);
return createRegionIterator(createRegionRelationsIterator(regionEdgesFamilyPath));
}
示例5: call
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<Tuple2<Integer, String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>> iter) throws Exception
{
List<Tuple2<Integer,String>> keyFileList = new ArrayList<>();
FileSystem fs = FileSystem.get(new Configuration());
// Form the filename for the exp table portion that corresponds to this partition
int taskId = TaskContext.getPartitionId();
logger.info("taskId = " + taskId);
String fileName = expOutDir + "/exp-" + String.format("%05d", taskId);
logger.info("fileName = " + fileName);
// Iterate over the elements of the partition
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(fileName), true)));
while (iter.hasNext())
{
// <queryHash, <<power>,<element^power mod N^2>>
Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>> expTuple = iter.next();
int queryHash = expTuple._1;
// Record the queryHash -> fileName
keyFileList.add(new Tuple2<>(queryHash, fileName));
// Write the partition elements to the corresponding exp table file
// each line: queryHash,<power>-<element^power mod N^2>
for (Tuple2<Integer,BigInteger> modPow : expTuple._2)
{
String lineOut = queryHash + "," + modPow._1 + "-" + modPow._2;
bw.write(lineOut);
bw.newLine();
}
}
bw.close();
return keyFileList.iterator();
}
示例6: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<WindowedValue<T>> compute(final Partition split,
final TaskContext context) {
final MetricsContainer metricsContainer = metricsAccum.localValue().getContainer(stepName);
@SuppressWarnings("unchecked")
final BoundedSource.BoundedReader<T> reader = createReader((SourcePartition<T>) split);
final Iterator<WindowedValue<T>> readerIterator =
new ReaderToIteratorAdapter<>(metricsContainer, reader);
return new InterruptibleIterator<>(context, JavaConversions.asScalaIterator(readerIterator));
}
示例7: addOnCompletition
import org.apache.spark.TaskContext; //导入依赖的package包/类
static void addOnCompletition(TaskContext taskContext, final Function0<?> function) {
taskContext.addOnCompleteCallback(new AbstractFunction0() {
@Override
public BoxedUnit apply() {
function.apply();
return BoxedUnit.UNIT;
}
});
}
示例8: buildWriter
import org.apache.spark.TaskContext; //导入依赖的package包/类
private ParquetWriter<Element> buildWriter(final String group, final String column, final boolean isEntity, final int splitNumber) throws IOException {
LOGGER.debug("Creating a new writer for group: {}", group + " with file number " + splitNumber);
final Path filePath = new Path(ParquetStore.getGroupDirectory(group, column,
tempFilesDir) + "/raw/split" + splitNumber + "/part-" + TaskContext.getPartitionId() + ".parquet");
return new ParquetElementWriter.Builder(filePath)
.isEntity(isEntity)
.withType(schemaUtils.getParquetSchema(group))
.usingConverter(schemaUtils.getConverter(group))
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
.withSparkSchema(schemaUtils.getSparkSchema(group))
.build();
}
示例9: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public scala.collection.Iterator<Map.Entry<Key, Value>> compute(final Partition split, final TaskContext context) {
final ByteArrayInputStream bais = new ByteArrayInputStream(serialisedConfiguration);
final Configuration configuration = new Configuration();
try {
configuration.readFields(new DataInputStream(bais));
bais.close();
} catch (final IOException e) {
throw new RuntimeException("IOException deserialising Configuration from byte array", e);
}
return new InterruptibleIterator<>(context,
JavaConversions.asScalaIterator(new RFileReaderIterator(split, context, configuration, auths)));
}
示例10: RFileReaderIterator
import org.apache.spark.TaskContext; //导入依赖的package包/类
public RFileReaderIterator(final Partition partition,
final TaskContext taskContext,
final Configuration configuration,
final Set<String> auths) {
this.partition = partition;
this.taskContext = taskContext;
this.configuration = configuration;
this.auths = auths;
try {
init();
} catch (final IOException e) {
throw new RuntimeException("IOException initialising RFileReaderIterator", e);
}
}
示例11: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<IndexedContent> compute(Partition split, TaskContext context) {
try {
Path path = new Path(partitions[split.index()].getPath());
java.util.Iterator<Content> iterator = new ContentIterator(path, NutchConfiguration.create(), contentTypeFilter);
return new IndexedContentIterator(iterator);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
示例12: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<Content> compute(Partition split, TaskContext context) {
try {
Path path = new Path(partitions[split.index()].getPath());
return new ContentIterator(path, NutchConfiguration.create(), contentTypeFilter);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
示例13: log
import org.apache.spark.TaskContext; //导入依赖的package包/类
private void log(String message, Object...params) {
if (Level.INFO.isGreaterOrEqual(log.getEffectiveLevel())) {
final int partitionId = TaskContext.getPartitionId();
final long threadId = Thread.currentThread().getId();
log.info("[" + threadId + ", PID=" + partitionId + "] " + String.format(message, params));
}
}
示例14: initPartition
import org.apache.spark.TaskContext; //导入依赖的package包/类
/**
* Looks up the extent of the current partition. If found, `match` method will
* activate the logic to avoid emitting duplicate join results from multiple partitions.
*
* Must be called before processing a partition. Must be called from the
* same instance that will be used to process the partition.
*/
protected void initPartition() {
if (dedupParams == null) {
return;
}
final int partitionId = TaskContext.getPartitionId();
final List<Envelope> partitionExtents = dedupParams.getPartitionExtents();
if (partitionId < partitionExtents.size()) {
extent = new HalfOpenRectangle(partitionExtents.get(partitionId));
} else {
log.warn("Didn't find partition extent for this partition: " + partitionId);
}
}
示例15: compute
import org.apache.spark.TaskContext; //导入依赖的package包/类
@Override
public Iterator<T> compute(Partition split, TaskContext context) {
initExtractorClient();
extractorClient.initIterator(split, config.getValue());
context.addTaskCompletionListener(new AbstractFunction1<TaskContext, BoxedUnit>() {
@Override
public BoxedUnit apply(TaskContext v1) {
extractorClient.close();
return null;
}
});
java.util.Iterator<T> iterator = new java.util.Iterator<T>() {
@Override
public boolean hasNext() {
return extractorClient.hasNext();
}
@Override
public T next() {
return extractorClient.next();
}
@Override
public void remove() {
throw new DeepIOException(
"Method not implemented (and won't be implemented anytime soon!!!)");
}
};
return new InterruptibleIterator<>(context, asScalaIterator(iterator));
}