Java Writables类代码示例

import org.apache.crunch.types.writable.Writables; //导入依赖的package包/类
public int run(String[] args) throws Exception {

        super.parseArguments(args);

        // Create an object to coordinate pipeline creation and execution.
        Pipeline pipeline = new MRPipeline(ComputeReadDepthInInterval.class, getConf());


        // Set up source to read from BAMs/SAMs
        TableSource<Long, SAMRecordWritable> samSource = From.formattedFile(inputPath,
                AnySAMInputFormat.class,
                Writables.longs(),
                Writables.writables(SAMRecordWritable.class));

        // Read in SAMRecords
        PCollection<SAMRecordWritable> records = pipeline.read(samSource).values();

        // Filter reads to mapped reads
        PCollection<SAMRecordWritable> mappedReads = records.filter(new MappedReadFilter());

        PCollection<Pair<String, Integer>> contigIntervals = mappedReads.parallelDo(
                "ComputeDepthInInterval",
                new ComputeDepthInInterval(intervalLength),
                Writables.pairs(Writables.strings(), Writables.ints()));

        // Compute read depth distribution
        PTable<Pair<String, Integer>, Long> contigIntervalCounts = contigIntervals.count();

        // Instruct the pipeline to write the resulting counts to a text file.
        pipeline.writeTextFile(contigIntervalCounts, outputPath);

        // Execute the pipeline as a MapReduce.
        PipelineResult result = pipeline.done();

        return result.succeeded() ? 0 : 1;
    }
 

import org.apache.crunch.types.writable.Writables; //导入依赖的package包/类
@Override
protected MRPipeline createPipeline() throws IOException {
  JobStepConfig stepConfig = getConfig();
  CovarianceSettings settings = CovarianceSettings.create();

  String instanceDir = stepConfig.getInstanceDir();
  long generationID = stepConfig.getGenerationID();
  String prefix = Namespaces.getInstanceGenerationPrefix(instanceDir, generationID);
  String outputKey = prefix + "covariance/";
  if (!validOutputPath(outputKey)) {
    return null;
  }

  KSketchIndex index = getCentersIndex(prefix);

  String inputKey = prefix + "normalized/";
  MRPipeline p = createBasicPipeline(CoMomentKeyFn.class);
  inputVectors(p, inputKey, MLAvros.vector()).parallelDo(
      "covAssign",
      new AssignFn<RealVector>(index, settings.useApprox()),
      Avros.tableOf(CKEY_PTYPE, Avros.pairs(MLAvros.vector(), Avros.doubles())))
      .parallelDo(
          "coMoment",
          new CoMomentKeyFn<ClusterKey>(CKEY_PTYPE),
          Avros.tableOf(Avros.pairs(CKEY_PTYPE, INDEX_PTYPE), COMOMENT_PTYPE))
      .groupByKey()
      .combineValues(new CoMomentAggregator())
      .parallelDo("covData", new CovarianceDataStringFn(), Writables.strings())
      .write(compressedTextOutput(p.getConfiguration(), outputKey));

  return p;
}
 

import org.apache.crunch.types.writable.Writables; //导入依赖的package包/类
public int run(String[] args) throws Exception {

        if (args.length != 3) {
            System.err.println("Usage: yarn jar varcrunch-*-SNAPSHOT-job.jar"
                                      + " [generic options] input output");
            System.err.println();
            GenericOptionsParser.printGenericCommandUsage(System.err);
            return 1;
        }

        String tumorReadsInputPath = args[0];
        String normalReadsInputPath = args[1];
        String outputPath = args[2];

        // Create an object to coordinate pipeline creation and execution.
        Pipeline pipeline = new MRPipeline(SomaticVarCrunch.class, getConf());


        // Set up source to read from BAMs/SAMs
        TableSource<Long, SAMRecordWritable> tumorInputSource = From.formattedFile(
                tumorReadsInputPath,
                AnySAMInputFormat.class,
                Writables.longs(),
                Writables.writables(SAMRecordWritable.class));

        TableSource<Long, SAMRecordWritable> normalInputSource = From.formattedFile(normalReadsInputPath,
                AnySAMInputFormat.class,
                Writables.longs(),
                Writables.writables(SAMRecordWritable.class));


        // Read in SAMRecords
        PCollection<SAMRecordWritable> tumorReads = pipeline.read(tumorInputSource).values();
        PCollection<SAMRecordWritable> normalReads = pipeline.read(normalInputSource).values();

        PCollection<Pair<String, Integer>> contigIntervals = tumorReads.parallelDo(
                new ComputeDepthInInterval(CONTIG_INTERVAL_SIZE),
                Writables.pairs(Writables.strings(), Writables.ints()));

        PTable<Pair<String, Integer>, Long> contigIntervalCounts = contigIntervals.count();

        pipeline.writeTextFile(contigIntervalCounts, outputPath);


        // Execute the pipeline as a MapReduce.
        PipelineResult result = pipeline.done();

        return result.succeeded() ? 0 : 1;
    }
 

import org.apache.crunch.types.writable.Writables; //导入依赖的package包/类
public int run(String[] args) throws Exception {

        super.parseArguments(args);

        // Create an object to coordinate pipeline creation and execution.
        Pipeline pipeline = new MRPipeline(GermlineVarCrunch.class, getConf());


        // Set up source to read from BAMs/SAMs
        TableSource<Long, SAMRecordWritable> readsInputSource = From.formattedFile(
                inputPath,
                AnySAMInputFormat.class,
                Writables.longs(),
                Writables.writables(SAMRecordWritable.class));

        // Read in SAMRecords
        PCollection<SAMRecordWritable> reads = pipeline.read(readsInputSource).values();

        PCollection<Pair<String, Integer>> contigIntervals = reads.parallelDo(
                new ComputeDepthInInterval(CONTIG_INTERVAL_SIZE),
                Writables.pairs(Writables.strings(), Writables.ints()));

        // Compute read depth distribution
        Map<Pair<String, Integer>, Long> contigIntervalCounts = contigIntervals.count().asMap().getValue();

        // Need to coalesce regions into evenly sized partitions
        Map<Pair<String, Integer>, Long> contigIntervalTasks = new HashMap<Pair<String, Integer>, Long>();

        // Flatmap reads to task
        // For now this is just distributes evenly across CONTIG_INTERVAL_SIZE splits
        PTable<Long, Pair<Integer, SAMRecordWritable>> partitionedReads = reads.parallelDo(
                "MapReadsToTask",
                new CollectNearbyReadsDoFn(CONTIG_INTERVAL_SIZE, contigIntervalTasks),
                Writables.tableOf(Writables.longs(), Writables.pairs(Writables.ints(), Writables.writables(SAMRecordWritable.class))));

        DoFn<Pair<Long, Iterable<Pair<Integer, SAMRecordWritable>>>, VariantContextWritable> variantCaller =
                new PileupGermlineVariants(CONTIG_INTERVAL_SIZE);

        PCollection<VariantContextWritable> variants = SecondarySort.sortAndApply(
                partitionedReads,
                variantCaller,
                Writables.writables(VariantContextWritable.class));

        pipeline.writeTextFile(variants, outputPath);

        // Execute the pipeline as a MapReduce.
        PipelineResult result = pipeline.done();

        return result.succeeded() ? 0 : 1;
    }