Java BucketUtils.openFile方法代码示例

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
protected void afterPipeline(Pipeline p) {
    bunny.stepEnd("dataflow");
    logger.info("Saving recalibration report to " + outputTablesPath);
    //  Get the table back and output it in text form to outputTablesPath.
    // TODO: if running on the cloud and the output destination is on the cloud, then it's faster to have a worker do it directly, without the file roundtrip.
    try (ObjectInputStream oin = new ObjectInputStream(BucketUtils.openFile(serializedOutputTablesPath, p.getOptions()))) {
        Object o = oin.readObject();
        RecalibrationTables rt = (RecalibrationTables) o;
        BaseRecalibratorFn.saveTextualReport(new File(outputTablesPath), readsHeader, rt,  recalArgs);
        bunny.stepEnd("repatriate_report");
    } catch (Exception e) {
        throw new GATKException("Unexpected: unable to read results file. (bug?)", e);
    }
    bunny.end();
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Override
protected void afterPipeline(Pipeline p) {
    bunny.stepEnd("dataflow");
    logger.info("Saving recalibration report to " + outputTablesPath);
    //  Get the table back and output it in text form to outputTablesPath
    // TODO: if running on the cloud and the output destination is on the cloud, then it's faster to have a worker do it directly, without the file roundtrip.
    try (ObjectInputStream oin = new ObjectInputStream(BucketUtils.openFile(serializedOutputTablesPath, p.getOptions()))) {
        Object o = oin.readObject();
        RecalibrationTables rt = (RecalibrationTables) o;
        BaseRecalibratorFn.saveTextualReport(new File(outputTablesPath), readsHeader, rt, recalArgs);
        bunny.stepEnd("repatriate_report");
    } catch (Exception e) {
        throw new GATKException("Unexpected: unable to read results file. (bug?)", e);
    }
    bunny.end();
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/** Read a file of contig names that will be ignored when checking for inter-contig pairs. */
private static Set<Integer> readCrossContigsToIgnoreFile( final String crossContigsToIgnoreFile,
                                                          final SAMSequenceDictionary dictionary ) {
    final Set<Integer> ignoreSet = new HashSet<>();
    try ( final BufferedReader rdr =
                  new BufferedReader(
                          new InputStreamReader(BucketUtils.openFile(crossContigsToIgnoreFile))) ) {
        String line;
        while ( (line = rdr.readLine()) != null ) {
            final int tigId = dictionary.getSequenceIndex(line);
            if ( tigId == -1 ) {
                throw new UserException("crossContigToIgnoreFile contains an unrecognized contig name: "+line);
            }
            ignoreSet.add(tigId);
        }
    }
    catch ( final IOException ioe ) {
        throw new UserException("Can't read crossContigToIgnore file "+crossContigsToIgnoreFile, ioe);
    }
    return ignoreSet;
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@VisibleForTesting static List<SVKmer> processFasta( final int kSize,
                                                     final int maxDUSTScore,
                                                     final String fastaFilename) {
    try ( BufferedReader rdr = new BufferedReader(new InputStreamReader(BucketUtils.openFile(fastaFilename))) ) {
        final List<SVKmer> kmers = new ArrayList<>((int) BucketUtils.fileSize(fastaFilename));
        String line;
        final StringBuilder sb = new StringBuilder();
        final SVKmer kmerSeed = new SVKmerLong();
        while ( (line = rdr.readLine()) != null ) {
            if ( line.charAt(0) != '>' ) sb.append(line);
            else if ( sb.length() > 0 ) {
                SVDUSTFilteredKmerizer.canonicalStream(sb,kSize,maxDUSTScore,kmerSeed).forEach(kmers::add);
                sb.setLength(0);
            }
        }
        if ( sb.length() > 0 ) {
            SVDUSTFilteredKmerizer.canonicalStream(sb,kSize,maxDUSTScore,kmerSeed).forEach(kmers::add);
        }
        return kmers;
    }
    catch ( IOException ioe ) {
        throw new GATKException("Can't read high copy kmers fasta file "+fastaFilename, ioe);
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Read a file of kmers.
 * Each line must be exactly
 * {@link org.broadinstitute.hellbender.tools.spark.sv.StructuralVariationDiscoveryArgumentCollection.FindBreakpointEvidenceSparkArgumentCollection#KMER_SIZE}
 * characters long, and must match [ACGT]*.
 */
public static Set<SVKmer> readKmersFile(final String kmersFilePath, final int kSize) {
    Utils.nonNull(kmersFilePath, "provided path for file containing kmers is null");
    Utils.validateArg(kSize > 0, "provided k-size is non positive: " + kSize);

    final Set<SVKmer> kmers;

    try ( final BufferedReader rdr =
                  new BufferedReader(new InputStreamReader(BucketUtils.openFile(kmersFilePath))) ) {
        final long fileLength = BucketUtils.fileSize(kmersFilePath);
        kmers = new HopscotchSet<>((int)(fileLength/(kSize+1)));
        String line;
        while ( (line = rdr.readLine()) != null ) {
            if ( line.length() != kSize ) {
                throw new GATKException("SVKmer kill set contains a line of length " + line.length() +
                        " but we were expecting K = " + kSize);
            }

            final SVKmerizer kmerizer = new SVKmerizer(line, kSize, 1, new SVKmerLong(kSize));
            if ( !kmerizer.hasNext() ) {
                throw new GATKException("Unable to kmerize the kmer kill set string '" + line + "'.");
            }

            kmers.add(kmerizer.next());
        }
    }
    catch ( final IOException ioe ) {
        throw new GATKException("Unable to read kmers from " + kmersFilePath, ioe);
    }

    return kmers;
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/** Read intervals from file. */
public static List<SVInterval> readIntervalsFile(final String intervalsFilePath,
                                                 final Map<String, Integer> contigNameMap ) {
    Utils.nonNull(intervalsFilePath, "provided intervals file path is null");
    Utils.nonNull(contigNameMap, "provided map for contig index lookup is null");

    final List<SVInterval> intervals;
    try ( final BufferedReader rdr =
                  new BufferedReader(new InputStreamReader(BucketUtils.openFile(intervalsFilePath))) ) {
        final long sizeGuess = BucketUtils.fileSize(intervalsFilePath)/25; // 25 is a guess on file line length
        intervals = new ArrayList<>((int)sizeGuess);
        String line;
        int lineNo = 0;
        while ( (line = rdr.readLine()) != null ) {
            ++lineNo;
            if (line.startsWith(REFERENCE_GAP_INTERVAL_FILE_COMMENT_LINE_PROMPT)) {
                continue;
            }
            final String[] tokens = line.split("\t");
            if ( tokens.length != 3 ) {
                throw new GATKException("Interval file " + intervalsFilePath + " line " +
                        lineNo + " did not contain 3 columns: " + line);
            }
            try {
                final Integer contigId = contigNameMap.get(tokens[0]);
                if ( contigId == null ) throw new GATKException("contig name " + tokens[0] + " not in dictionary");
                final int start = Integer.valueOf(tokens[1]);
                final int end = Integer.valueOf(tokens[2]);
                intervals.add(new SVInterval(contigId, start, end));
            }
            catch ( final Exception e ) {
                throw new GATKException("Unable to parse interval file " + intervalsFilePath + " line " + lineNo + ": " + line, e);
            }
        }
    }
    catch ( final IOException ioe ) {
        throw new GATKException("Unable to read intervals from " + intervalsFilePath, ioe);
    }
    return intervals;
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
private Set<String> parseReadNames() {

        try ( final BufferedReader rdr =
                      new BufferedReader(new InputStreamReader(BucketUtils.openFile(readNameFile))) ) {
            return rdr.lines().map(s -> s.replace("^@", "")
                                         .replace("/1$", "")
                                         .replace("/2$", ""))
                    .collect(Collectors.toCollection(HashSet::new));
        } catch ( final IOException ioe ) {
            throw new GATKException("Unable to read names file from " + readNameFile, ioe);
        }
    }
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
public static PSKmerCollection readKmerFilter(final String uri) {
    final Input input = new Input(BucketUtils.openFile(uri));
    final Kryo kryo = new Kryo();
    if (uri.endsWith(HOPSCOTCH_SET_EXTENSION)) {
        return kryo.readObject(input, PSKmerSet.class);
    } else if (uri.endsWith(BLOOM_FILTER_EXTENSION)) {
        return kryo.readObject(input, PSKmerBloomFilter.class);
    }
    throw new UserException.BadInput("Unknown kmer set extension in file name " + uri);
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Reads taxonomy database that has been serialized to a file
 */
@SuppressWarnings("unchecked")
public static PSTaxonomyDatabase readTaxonomyDatabase(final String filePath) {
    final Kryo kryo = new Kryo();
    kryo.setReferences(false);
    final Input input = new Input(BucketUtils.openFile(filePath));
    final PSTaxonomyDatabase taxonomyDatabase = kryo.readObject(input, PSTaxonomyDatabase.class);
    input.close();
    return taxonomyDatabase;
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
@Test(groups = {"bucket"})
public void testLoadFastaDictionaryFromGCSBucket() throws IOException {
    final String bucketDictionary = getGCPTestInputPath() + "org/broadinstitute/hellbender/utils/ReferenceUtilsTest.dict";

    try ( final InputStream referenceDictionaryStream = BucketUtils.openFile(bucketDictionary) ) {
        final SAMSequenceDictionary dictionary = ReferenceUtils.loadFastaDictionary(referenceDictionaryStream);

        Assert.assertNotNull(dictionary, "Sequence dictionary null after loading");
        Assert.assertEquals(dictionary.size(), 4, "Wrong sequence dictionary size after loading");
    }
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
public static List<FastqRead> readFastqFile( final String fileName ) {
    final int INITIAL_CAPACITY = 10000; // absolute guess, just something not too crazy small
    final List<FastqRead> reads = new ArrayList<>(INITIAL_CAPACITY);
    try ( final BufferedReader reader = new BufferedReader(new InputStreamReader(BucketUtils.openFile(fileName))) ) {
        String header;
        int lineNo = 0;
        while ( (header = reader.readLine()) != null ) {
            lineNo += 1;
            if (!FASTQ_READ_HEADER_PATTERN.matcher(header).find()) {
                throw new GATKException("In FASTQ file "+fileName+" sequence identifier line does not start with @ on line "+lineNo);
            }
            final String callLine = reader.readLine();
            lineNo += 1;
            if ( callLine == null ) {
                throw new GATKException("In FASTQ file "+fileName+" file truncated: missing calls.");
            }
            final String sepLine = reader.readLine();
            lineNo += 1;
            if ( sepLine == null ) {
                throw new GATKException("In FASTQ file "+fileName+" file truncated: missing + line.");
            }
            if ( sepLine.length() < 1 || sepLine.charAt(0) != LINE_SEPARATOR_CHR ) {
                throw new GATKException("In FASTQ file " + fileName + " separator line does not start with + on line " + lineNo);
            }
            final String qualLine = reader.readLine();
            lineNo += 1;
            if ( qualLine == null ) {
                throw new UserException.BadInput("In FASTQ file "+fileName+" file truncated: missing quals.");
            }
            if ( callLine.length() != qualLine.length() ) {
                throw new UserException.BadInput("In FASTQ file "+fileName+" there are "+qualLine.length()+
                        " quality scores on line "+lineNo+" but there are "+callLine.length()+" base calls.");
            }
            final byte[] quals = qualLine.getBytes();
            final byte[] calls = callLine.getBytes();
            reads.add(new FastqRead(header, calls, quals));
        }
    }
    catch ( final IOException ioe ) {
        throw new GATKException("Can't read "+fileName, ioe);
    }
    return reads;
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Takes a URI string (a local file, a Google bucket file, or an HDFS file) and returns a {@link Reader}
 *
 * @param path input URI string
 * @return an instance of {@link Reader}
 */
private Reader getReaderFromURI(@Nonnull final String path) {
    final InputStream inputStream = BucketUtils.openFile(path);
    return new BufferedReader(new InputStreamReader(inputStream));
}
 

import org.broadinstitute.hellbender.utils.gcs.BucketUtils; //导入方法依赖的package包/类
/**
 * Create a new GATKReport with the contents of a GATKReport on disk.
 *
 * @param filename the path to the file to load
 */
public GATKReport(String filename) {
    this(BucketUtils.openFile(filename));
}