Java BucketUtils.createFile方法代码示例

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Writes accessions contained in a SAM header that do not exist in the taxonomy database
 */
public static void writeMissingReferenceAccessions(final String path, final SAMFileHeader header, final PSTaxonomyDatabase taxDB,
                                                   final Logger logger) {
    if (header != null && header.getSequenceDictionary() != null && header.getSequenceDictionary().getSequences() != null) {
        final Set<String> unknownSequences = header.getSequenceDictionary().getSequences().stream()
                .map(SAMSequenceRecord::getSequenceName)
                .filter(name -> !taxDB.accessionToTaxId.containsKey(name))
                .collect(Collectors.toSet());
        try (final PrintStream file = new PrintStream(BucketUtils.createFile(path))) {
            unknownSequences.stream().forEach(file::print);
            if (file.checkError()) {
                logger.warn("Error writing to header warnings file");
            }
        }
    }

}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Output a tab-delimited table of taxonomic scores
 */
public static void writeScoresFile(final Map<Integer, PSPathogenTaxonScore> scores,
                                   final PSTree tree, final String filePath) {
    final String header = "tax_id\ttaxonomy\ttype\tname\t" + PSPathogenTaxonScore.outputHeader;
    try (final PrintStream printStream = new PrintStream(BucketUtils.createFile(filePath))) {
        printStream.println(header);
        for (final int key : scores.keySet()) {
            final String name = tree.getNameOf(key);
            final String rank = tree.getRankOf(key);
            final List<String> path = tree.getPathOf(key).stream().map(tree::getNameOf).collect(Collectors.toList());
            Collections.reverse(path);
            final String taxonomy = String.join("|", path);
            final String line = key + "\t" + taxonomy + "\t" + rank + "\t" + name + "\t" + scores.get(key).toString(tree);
            printStream.println(line.replace(" ", "_"));
        }
        if (printStream.checkError()) {
            throw new UserException.CouldNotCreateOutputFile(filePath, new IOException());
        }
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
protected void runTool( JavaSparkContext ctx ) {
    if (joinStrategy == JoinStrategy.BROADCAST && ! getReference().isCompatibleWithSparkBroadcast()){
        throw new UserException.Require2BitReferenceForBroadcast();
    }

    if (joinStrategy == JoinStrategy.OVERLAPS_PARTITIONER && getHeaderForReads().getSortOrder() != SAMFileHeader.SortOrder.coordinate) {
        throw new UserException.BadInput("Reads must be coordinate sorted when using the overlaps partitioner join strategy.");
    }

    JavaRDD<GATKRead> initialReads = getReads();
    VariantsSparkSource variantsSparkSource = new VariantsSparkSource(ctx);
    JavaRDD<GATKVariant> bqsrKnownVariants = variantsSparkSource.getParallelVariants(knownVariants, getIntervals());

    // TODO: Look into broadcasting the reference to all of the workers. This would make AddContextDataToReadSpark
    // TODO: and ApplyBQSRStub simpler (#855).
    JavaPairRDD<GATKRead, ReadContextData> rddReadContext = AddContextDataToReadSpark.add(ctx, initialReads, getReference(), bqsrKnownVariants, knownVariants, joinStrategy, getHeaderForReads().getSequenceDictionary(), readShardSize, readShardPadding);

    // TODO: broadcast the reads header?
    final RecalibrationReport bqsrReport = BaseRecalibratorSparkFn.apply(rddReadContext, getHeaderForReads(), getReferenceSequenceDictionary(), bqsrArgs);

    try ( final PrintStream reportStream = new PrintStream(BucketUtils.createFile(outputTablesPath)) ) {
        RecalUtils.outputRecalibrationReport(reportStream, bqsrArgs, bqsrReport.getQuantizationInfo(), bqsrReport.getRecalibrationTables(), bqsrReport.getCovariates());
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
public void processElement(ProcessContext c) throws Exception {
    String dest = c.element();
    logger.info("Saving to " + dest);
    Iterable<GATKRead> reads = c.sideInput(iterableView);
    OutputStream outputStream = BucketUtils.createFile(dest, c.getPipelineOptions());
    try (SAMFileWriter writer = new SAMFileWriterFactory().makeBAMWriter(header, false, outputStream)) {
        for (GATKRead r : reads) {
            final SAMRecord sr = r.convertToSAMRecord(header);
            writer.addAlignment(sr);
        }
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Write a {@link MetricsFile} to the given path, can be any destination supported by {@link BucketUtils#createFile(String)}
 * @param metricsFile a {@link MetricsFile} object to write to disk
 * @param metricsOutputPath the path (or uri) to write the metrics to
 */
public static void saveMetrics(final MetricsFile<?, ?> metricsFile, String metricsOutputPath) {
    try(PrintWriter out = new PrintWriter(BucketUtils.createFile(metricsOutputPath))) {
        metricsFile.write(out);
    } catch (SAMException e ){
        throw new UserException.CouldNotCreateOutputFile("Could not write metrics to file: " + metricsOutputPath, e);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
public static void saveTextualReport(String output, SAMFileHeader header, RecalibrationTables rt, RecalibrationArgumentCollection recalArgs) throws IOException {
    OutputStream oStream = BucketUtils.createFile(output);
    QuantizationInfo qi = new QuantizationInfo(rt, recalArgs.QUANTIZING_LEVELS);
    if (recalArgs.FORCE_PLATFORM != null) {
        recalArgs.DEFAULT_PLATFORM = recalArgs.FORCE_PLATFORM;
    }
    StandardCovariateList covariates = new StandardCovariateList(recalArgs, header);
    try ( PrintStream reportStream = new PrintStream(oStream) ) {
        RecalUtils.outputRecalibrationReport(reportStream, recalArgs, qi, rt, covariates);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/** Write a list of FASTQ records into a file. */
public static void writeFastqFile(final String fileName, final Iterator<FastqRead> fastqReadItr ) {
    try ( final OutputStream writer =
                  new BufferedOutputStream(BucketUtils.createFile(fileName)) ) {
        writeFastqStream(writer, fastqReadItr);
    } catch ( final IOException ioe ) {
        throw new GATKException("Can't write "+fileName, ioe);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
private static void writeVariants(final String fileName, final List<VariantContext> variantsArrayList,
                                  final SAMSequenceDictionary referenceSequenceDictionary) {
    try (final OutputStream outputStream
                 = new BufferedOutputStream(BucketUtils.createFile(fileName))) {

        final VariantContextWriter vcfWriter = getVariantContextWriter(outputStream, referenceSequenceDictionary);

        vcfWriter.writeHeader(getVcfHeader(referenceSequenceDictionary));
        variantsArrayList.forEach(vcfWriter::add);
        vcfWriter.close();

    } catch (final IOException e) {
        throw new GATKException("Could not create output file", e);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/** Write kmers to file. */
public static <KType extends SVKmer> void writeKmersFile(final String kmersFilePath, final int kSize,
                                                         final Collection<KType> kmers) {
    try ( final Writer writer =
                  new BufferedWriter(new OutputStreamWriter(BucketUtils.createFile(kmersFilePath))) ) {
        for ( final KType kmer : kmers ) {
            writer.write(kmer.toString(kSize));
            writer.write('\n');
        }
    }
    catch ( final IOException ioe ) {
        throw new GATKException("Unable to write kmers to " + kmersFilePath, ioe);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/** Write intervals to a file. */
public static void writeIntervalsFile( final String intervalsFilePath,
                                       final Collection<SVInterval> intervals, final List<String> contigNames ) {
    try (final OutputStreamWriter writer = new OutputStreamWriter(new BufferedOutputStream(
            BucketUtils.createFile(intervalsFilePath)))) {
        for (final SVInterval interval : intervals) {
            final String seqName = contigNames.get(interval.getContig());
            writer.write(seqName + "\t" + interval.getStart() + "\t" + interval.getEnd() + "\n");
        }
    } catch (final IOException ioe) {
        throw new GATKException("Can't write intervals file " + intervalsFilePath, ioe);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
public AlignedAssemblyOrExcuse apply( final Tuple2<Integer, List<SVFastqUtils.FastqRead>> intervalAndReads ) {
    final int intervalID = intervalAndReads._1();
    final List<SVFastqUtils.FastqRead> readsList = intervalAndReads._2();

    final int fastqSize = readsList.stream().mapToInt(FastqRead -> FastqRead.getBases().length).sum();
    if (fastqSize > maxFastqSize) {
        return new AlignedAssemblyOrExcuse(intervalID, "no assembly -- too big (" + fastqSize + " bytes).");
    }

    if ( fastqDir != null ) {
        final String fastqName = String.format("%s/%s.fastq", fastqDir, AlignedAssemblyOrExcuse.formatAssemblyID(intervalID));
        final ArrayList<SVFastqUtils.FastqRead> sortedReads = new ArrayList<>(readsList);
        sortedReads.sort(Comparator.comparing(SVFastqUtils.FastqRead::getHeader));
        SVFastqUtils.writeFastqFile(fastqName, sortedReads.iterator());
    }
    final long timeStart = System.currentTimeMillis();
    final FermiLiteAssembly assembly = new FermiLiteAssembler().createAssembly(readsList);
    final int secondsInAssembly = (int)((System.currentTimeMillis() - timeStart + 500)/1000);
    if ( fastqDir != null && writeGFAs ) {
        final String gfaName =  String.format("%s/%s.gfa",fastqDir, AlignedAssemblyOrExcuse.formatAssemblyID(intervalID));
        try ( final OutputStream os = BucketUtils.createFile(gfaName) ) {
            assembly.writeGFA(os);
        }
        catch ( final IOException ioe ) {
            throw new GATKException("Can't write "+gfaName, ioe);
        }
    }
    final List<byte[]> tigSeqs =
            assembly.getContigs().stream()
                    .map(FermiLiteAssembly.Contig::getSequence)
                    .collect(SVUtils.arrayListCollector(assembly.getNContigs()));
    try ( final BwaMemAligner aligner = new BwaMemAligner(BwaMemIndexCache.getInstance(alignerIndexFile)) ) {
        aligner.setIntraCtgOptions();
        final List<List<BwaMemAlignment>> alignments = aligner.alignSeqs(tigSeqs);
        return new AlignedAssemblyOrExcuse(intervalID, assembly, secondsInAssembly, alignments);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * write template names and interval IDs to a file.
 */
public static void writeQNames( final String qNameFile,
                                final Iterable<QNameAndInterval> qNames ) {
    try ( final OutputStreamWriter writer =
                  new OutputStreamWriter(new BufferedOutputStream(BucketUtils.createFile(qNameFile))) ) {
        for ( final QNameAndInterval qnameAndInterval : qNames ) {
            writer.write(qnameAndInterval.toString() + "\n");
        }
    } catch ( final IOException ioe ) {
        throw new GATKException("Can't write qname intervals file " + qNameFile, ioe);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
private static final Tuple2<Integer, String> readChunkToHdfs(final String inputGCSPathFinal, final long chunkSize, final Integer chunkNum, final String outputDirectory) {
    final Path gcsPath = IOUtils.getPath(inputGCSPathFinal);
    final String basename = gcsPath.getName(gcsPath.getNameCount() - 1).toString();
    org.apache.hadoop.fs.Path outputPath = new org.apache.hadoop.fs.Path(outputDirectory);
    final String chunkPath = outputPath + "/" + basename + ".chunk." + chunkNum;

    try (SeekableByteChannel channel = Files.newByteChannel(gcsPath);
         final OutputStream outputStream = new BufferedOutputStream(BucketUtils.createFile(chunkPath))){

        final long start = chunkSize * (long) chunkNum;
        channel.position(start);
        ByteBuffer byteBuffer = ByteBuffer.allocateDirect((int) Math.min(SIXTY_FOUR_MIB, chunkSize));
        long bytesRead = 0;
        while(channel.read(byteBuffer) > 0) {
            byteBuffer.flip();
            while (byteBuffer.hasRemaining() && bytesRead < chunkSize) {
                byte b = byteBuffer.get();
                outputStream.write(b);
                bytesRead++;
            }
            if (bytesRead == chunkSize) {
                break;
            }
            if (bytesRead > chunkSize) {
                throw new GATKException("Encountered an unknown error condition and read too many bytes; output file may be corrupt");
            }
            byteBuffer.clear();
        }
    } catch (IOException e) {
        throw new GATKException(e.getMessage() + "; inputGCSPathFinal = " + inputGCSPathFinal, e);
    }
    return new Tuple2<>(chunkNum, chunkPath);
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
protected void runTool(final JavaSparkContext ctx) {
    final JavaRDD<GATKRead> reads = getReads();

    final FlagStatus result = reads.aggregate(new FlagStatus(), FlagStatus::add, FlagStatus::merge);
    System.out.println(result);

    if(out != null ) {
        try ( final PrintStream ps = new PrintStream(BucketUtils.createFile(out)) ) {
            ps.print(result);
        }
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
protected void runTool(final JavaSparkContext ctx) {
    final VariantsSparkSource vss = new VariantsSparkSource(ctx);
    final JavaRDD<VariantContext> variants = vss.getParallelVariantContexts(input, getIntervals());

    final long count = variants.count();
    System.out.println(count);

    if( out != null) {
        try (final PrintStream ps = new PrintStream(BucketUtils.createFile(out))) {
            ps.print(count);
        }
    }
}