Java SparkFiles类代码示例

import org.apache.spark.SparkFiles; //导入依赖的package包/类
void loadNativeLib(){
    if(nativeLibPath.contains("/"))
        System.load(nativeLibPath);
    else
        try{
            System.load(SparkFiles.get(nativeLibPath));
        } catch(Exception e){
            System.load(nativeLibPath);
        }
}
 

import org.apache.spark.SparkFiles; //导入依赖的package包/类
/**
 * Imitates map part of hadoop streaming job.
 * It executes provided script for every key in inputRecords rdd.
 * <br/><br/>
 * It is assumed that provided script will read records from standard input (one line for one record)
 * and write mapped record into standard output (also one line for one record).
 * Mapped record can be a key/value pair. In that case script should return key and value
 * splitted by tab (\t) character in single line. 
 */
public JavaPairRDD<String, String> doMap(JavaPairRDD<AvroKey<GenericRecord>, NullWritable> inputRecords, String scriptName, String args) {

	JavaRDD<String> mappedRecords = inputRecords.keys().pipe("python " + SparkFiles.get(scriptName) + " " + args);

	JavaPairRDD<String, String> outputRecords = mappedRecords
			.mapToPair(line -> {
				String[] splittedPair = line.split("\t");
				return new Tuple2<String, String>(splittedPair[0], (splittedPair.length == 1) ? null : splittedPair[1]);
			});

	return outputRecords;
}
 

import org.apache.spark.SparkFiles; //导入依赖的package包/类
/**
 * Imitates reduce part of hadoop streaming job.
 * <br/><br/>
 * It is assumed that provided script will read records from standard input (one line for one record)
 * and group records with the same key into single record (reduce).
 * Method assures that all input records with the same key will be transfered in adjacent lines.
 * Reduced records should be written by script into standard output (one line for one record).
 * Reduced records must be json strings of class provided as argument.
 */
public JavaPairRDD<AvroKey<GenericRecord>, NullWritable> doReduce(JavaPairRDD<String, String> inputRecords, String scriptName, String args, Class<? extends GenericRecord> outputClass) {

	JavaRDD<String> reducedRecords = inputRecords.sortByKey()
			.map(record -> record._1 + ((record._2 == null) ? "" : ("\t" + record._2)))
			.pipe("python " + SparkFiles.get(scriptName) + " " + args);

	JavaPairRDD<AvroKey<GenericRecord>, NullWritable> outputRecords = reducedRecords
			.map(recordString -> AvroGsonFactory.create().fromJson(recordString, outputClass))
			.mapToPair(record -> new Tuple2<AvroKey<GenericRecord>, NullWritable>(new AvroKey<>(record), NullWritable.get()));

	return outputRecords;
}
 

import org.apache.spark.SparkFiles; //导入依赖的package包/类
/**
 * Performs read alignment on a RDD.
 * @param unalignedReads the reads to align.
 * @param pairedAlignment whether it should perform pair-end alignment ({@code true}) or single-end alignment ({@code false}).
 * @return never {@code null}.
 */
public JavaRDD<GATKRead> align(final JavaRDD<GATKRead> unalignedReads, final boolean pairedAlignment) {
    final Broadcast<SAMFileHeader> broadcastHeader = this.broadcastHeader;
    final String indexFileName = this.indexFileName;
    final boolean resolveIndexFileName = this.resolveIndexFileName;
    return unalignedReads.mapPartitions(itr ->
            new ReadAligner(resolveIndexFileName ? SparkFiles.get(indexFileName) : indexFileName, broadcastHeader.value(), pairedAlignment).apply(itr));
}
 

import org.apache.spark.SparkFiles; //导入依赖的package包/类
private static String getScriptPath() {
    
    String path = SparkFiles.get("scripts");
    
    if (SystemUtils.IS_OS_WINDOWS) {
        return path.replace("\\", "/");
    }
    
    return path;
}