Java SortingCollection类代码示例

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
private synchronized void addRecord(final String barcode, final CLUSTER_OUTPUT_RECORD record) {
    // Grab the existing collection, or initialize it if it doesn't yet exist
    SortingCollection<CLUSTER_OUTPUT_RECORD> recordCollection = this.barcodeToRecordCollection.get(barcode);
    if (recordCollection == null) {
        // TODO: The implementation here for supporting ignoreUnexpectedBarcodes is not efficient,
        // but the alternative is an extensive rewrite.  We are living with the inefficiency for
        // this special case for the time being.
        if (!barcodeRecordWriterMap.containsKey(barcode)) {
            if (ignoreUnexpectedBarcodes) {
                return;
            }
            throw new PicardException(String.format("Read records with barcode %s, but this barcode was not expected.  (Is it referenced in the parameters file?)", barcode));
        }
        recordCollection = newSortingCollection();
        this.barcodeToRecordCollection.put(barcode, recordCollection);
    }
    recordCollection.add(record);
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * @param basecallsDir             Where to read basecalls from.
 * @param lane                     What lane to process.
 * @param readStructure            How to interpret each cluster.
 * @param barcodeRecordWriterMap   Map from barcode to CLUSTER_OUTPUT_RECORD writer.  If demultiplex is false, must contain
 *                                 one writer stored with key=null.
 * @param demultiplex              If true, output is split by barcode, otherwise all are written to the same output stream.
 * @param maxReadsInRamPerTile     Configures number of reads each tile will store in RAM before spilling to disk.
 * @param tmpDirs                  For SortingCollection spilling.
 * @param numProcessors            Controls number of threads.  If <= 0, the number of threads allocated is
 *                                 available cores - numProcessors.
 * @param forceGc                  Force explicit GC periodically.  This is good for causing memory maps to be released.
 * @param firstTile                (For debugging) If non-null, start processing at this tile.
 * @param tileLimit                (For debugging) If non-null, process no more than this many tiles.
 * @param outputRecordComparator   For sorting output records within a single tile.
 * @param codecPrototype           For spilling output records to disk.
 * @param outputRecordClass        Inconveniently needed to create SortingCollections.
 * @param includeNonPfReads        If true, will include ALL reads (including those which do not have PF set)
 * @param ignoreUnexpectedBarcodes If true, will ignore reads whose called barcode is not found in barcodeRecordWriterMap,
 *                                 otherwise will throw an exception
 */
public IlluminaBasecallsConverter(final File basecallsDir, final int lane, final ReadStructure readStructure,
                                  final Map<String, ? extends ConvertedClusterDataWriter<CLUSTER_OUTPUT_RECORD>> barcodeRecordWriterMap,
                                  final boolean demultiplex,
                                  final int maxReadsInRamPerTile,
                                  final List<File> tmpDirs,
                                  final int numProcessors, final boolean forceGc,
                                  final Integer firstTile, final Integer tileLimit,
                                  final Comparator<CLUSTER_OUTPUT_RECORD> outputRecordComparator,
                                  final SortingCollection.Codec<CLUSTER_OUTPUT_RECORD> codecPrototype,
                                  final Class<CLUSTER_OUTPUT_RECORD> outputRecordClass,
                                  final BclQualityEvaluationStrategy bclQualityEvaluationStrategy,
                                  final boolean applyEamssFiltering,
                                  final boolean includeNonPfReads,
                                  final boolean ignoreUnexpectedBarcodes
) {
    this(basecallsDir, null, lane, readStructure,
            barcodeRecordWriterMap, demultiplex, maxReadsInRamPerTile,
            tmpDirs, numProcessors, forceGc, firstTile, tileLimit,
            outputRecordComparator, codecPrototype, outputRecordClass,
            bclQualityEvaluationStrategy, applyEamssFiltering,
            includeNonPfReads, ignoreUnexpectedBarcodes);
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * Adds the provided record to this tile.
 */
public synchronized void addRecord(final String barcode, final CLUSTER_OUTPUT_RECORD record) {
    this.recordCount += 1;

    // Grab the existing collection, or initialize it if it doesn't yet exist
    SortingCollection<CLUSTER_OUTPUT_RECORD> recordCollection = this.barcodeToRecordCollection.get(barcode);
    if (recordCollection == null) {
        // TODO: The implementation here for supporting ignoreUnexpectedBarcodes is not efficient,
        // but the alternative is an extensive rewrite.  We are living with the inefficiency for
        // this special case for the time being.
        if (!barcodeRecordWriterMap.containsKey(barcode)) {
            if (ignoreUnexpectedBarcodes) {
                return;
            }
            throw new PicardException(String.format("Read records with barcode %s, but this barcode was not expected.  (Is it referenced in the parameters file?)", barcode));
        }
        recordCollection = this.newSortingCollection();
        this.barcodeToRecordCollection.put(barcode, recordCollection);
        this.barcodeToProcessingState.put(barcode, null);
    }
    recordCollection.add(record);
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
RevertSamSorter(
        final boolean outputByReadGroup,
        final Map<String, SAMFileHeader> headerMap,
        final SAMFileHeader singleOutHeader,
        final int maxRecordsInRam) {

    this.outputByReadGroup = outputByReadGroup;
    if (outputByReadGroup) {
        for (final Map.Entry<String, SAMFileHeader> entry : headerMap.entrySet()) {
            final String readGroupId = entry.getKey();
            final SAMFileHeader outHeader = entry.getValue();
            final SortingCollection<SAMRecord> sorter = SortingCollection.newInstance(SAMRecord.class, new BAMRecordCodec(outHeader), new SAMRecordQueryNameComparator(), maxRecordsInRam);
            sorterMap.put(readGroupId, sorter);
        }
        singleSorter = null;
    } else {
        singleSorter = SortingCollection.newInstance(SAMRecord.class, new BAMRecordCodec(singleOutHeader), new SAMRecordQueryNameComparator(), maxRecordsInRam);
    }
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * Merge the inputs and sort them by adding each input's content to a single SortingCollection.
 * <p/>
 * NB: It would be better to have a merging iterator as in MergeSamFiles, as this would perform better for pre-sorted inputs.
 * Here, we are assuming inputs are unsorted, and so adding their VariantContexts iteratively is fine for now.
 * MergeVcfs exists for simple merging of presorted inputs.
 *
 * @param readers      - a list of VCFFileReaders, one for each input VCF
 * @param outputHeader - The merged header whose information we intend to use in the final output file
 */
private SortingCollection<VariantContext> sortInputs(final List<VCFFileReader> readers, final VCFHeader outputHeader) {
    final ProgressLogger readProgress = new ProgressLogger(log, 25000, "read", "records");

    // NB: The default MAX_RECORDS_IN_RAM may not be appropriate here. VariantContexts are smaller than SamRecords
    // We would have to play around empirically to find an appropriate value. We are not performing this optimization at this time.
    final SortingCollection<VariantContext> sorter =
            SortingCollection.newInstance(
                    VariantContext.class,
                    new VCFRecordCodec(outputHeader, VALIDATION_STRINGENCY != ValidationStringency.STRICT),
                    outputHeader.getVCFRecordComparator(),
                    MAX_RECORDS_IN_RAM,
                    TMP_DIR);
    int readerCount = 1;
    for (final VCFFileReader reader : readers) {
        log.info("Reading entries from input file " + readerCount);
        for (final VariantContext variantContext : reader) {
            sorter.add(variantContext);
            readProgress.record(variantContext.getContig(), variantContext.getStart());
        }
        reader.close();
        readerCount++;
    }
    return sorter;
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
@Override
public SortingCollection.Codec<SortableRecord> clone() {
    try {
        return (SortingCollection.Codec<SortableRecord>) super.clone();
    } catch (CloneNotSupportedException e) {
        throw new SortingException(e);
    }
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * @param maxMemory - in megabytes
 * @throws IOException
 *
 */
public void run(int maxMemory) throws IOException {
    try (
            PrintWriter writer = NgbFileUtils.isGzCompressed(outputFile.getName()) ?
                    new PrintWriter(new OutputStreamWriter(new BlockCompressedOutputStream(outputFile), UTF_8)) :
                    new PrintWriter(outputFile, UTF_8);
            AsciiLineReader reader = NgbFileUtils.isGzCompressed(outputFile.getName()) ?
                    new AsciiLineReader(new BlockCompressedInputStream(inputFile)) :
                    new AsciiLineReader(new FileInputStream(inputFile))
    ) {

        SortableRecordCodec codec = new SortableRecordCodec();

        SortingCollection cltn = SortingCollection.newInstance(
                SortableRecord.class, codec, comparator, maxMemory * 1024 * 1024 / ESTIMATED_RECORD_SIZE, tmpDir);

        Parser parser = getParser();

        String firstDataRow = writeHeader(reader, writer);
        if (firstDataRow != null && !firstDataRow.isEmpty()) {
            cltn.add(parser.createRecord(firstDataRow));
        }

        SortableRecord next;
        while ((next = parser.readNextRecord(reader)) != null) {
            cltn.add(next);
        }

        CloseableIterator<SortableRecord> iter = cltn.iterator();
        while (iter.hasNext()) {
            SortableRecord al = iter.next();
            writer.println(al.getText());

        }
        iter.close();
    }
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * @param maxMemory - in megabytes
 * @throws IOException
 *
 */
public void run(int maxMemory) throws IOException {
    try (
            PrintWriter writer = NgbFileUtils.isGzCompressed(outputFile.getName()) ?
                    new PrintWriter(new OutputStreamWriter(new BlockCompressedOutputStream(outputFile), UTF_8)) :
                    new PrintWriter(outputFile, UTF_8);
            AsciiLineReader reader = NgbFileUtils.isGzCompressed(inputFile.getName()) ?
                    new AsciiLineReader(new BlockCompressedInputStream(inputFile)) :
                    new AsciiLineReader(new FileInputStream(inputFile))
    ) {

        SortableRecordCodec codec = new SortableRecordCodec();

        SortingCollection cltn = SortingCollection.newInstance(
                SortableRecord.class, codec, comparator, maxMemory * 1024 * 1024 / ESTIMATED_RECORD_SIZE, tmpDir);

        Parser parser = getParser();

        String firstDataRow = writeHeader(reader, writer);
        if (firstDataRow != null && !firstDataRow.isEmpty()) {
            cltn.add(parser.createRecord(firstDataRow));
        }

        SortableRecord next;
        while ((next = parser.readNextRecord(reader)) != null) {
            cltn.add(next);
        }

        CloseableIterator<SortableRecord> iter = cltn.iterator();
        while (iter.hasNext()) {
            SortableRecord al = iter.next();
            writer.println(al.getText());

        }
        iter.close();
    }
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
private synchronized SortingCollection<CLUSTER_OUTPUT_RECORD> newSortingCollection() {
    final int maxRecordsInRam =
            Math.max(1, maxReadsInRamPerTile /
                    barcodeRecordWriterMap.size());
    return SortingCollection.newInstance(
            outputRecordClass,
            codecPrototype.clone(),
            outputRecordComparator,
            maxRecordsInRam,
            tmpDirs);
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
/**
 * @param barcodeRecordWriterMap   Map from barcode to CLUSTER_OUTPUT_RECORD writer.  If demultiplex is false, must contain
 *                                 one writer stored with key=null.
 * @param demultiplex              If true, output is split by barcode, otherwise all are written to the same output stream.
 * @param maxReadsInRamPerTile     Configures number of reads each tile will store in RAM before spilling to disk.
 * @param tmpDirs                  For SortingCollection spilling.
 * @param numProcessors            Controls number of threads.  If <= 0, the number of threads allocated is
 *                                 available cores - numProcessors.
 * @param outputRecordComparator   For sorting output records within a single tile.
 * @param codecPrototype           For spilling output records to disk.
 * @param outputRecordClass        Inconveniently needed to create SortingCollections.
 * @param ignoreUnexpectedBarcodes If true, will ignore reads whose called barcode is not found in barcodeRecordWriterMap,
 */
BasecallsConverter(final Map<String, ? extends ConvertedClusterDataWriter<CLUSTER_OUTPUT_RECORD>> barcodeRecordWriterMap,
                   final int maxReadsInRamPerTile,
                   final List<File> tmpDirs,
                   final SortingCollection.Codec<CLUSTER_OUTPUT_RECORD> codecPrototype,
                   final boolean ignoreUnexpectedBarcodes,
                   final boolean demultiplex,
                   final Comparator<CLUSTER_OUTPUT_RECORD> outputRecordComparator,
                   final BclQualityEvaluationStrategy bclQualityEvaluationStrategy,
                   final Class<CLUSTER_OUTPUT_RECORD> outputRecordClass,
                   final int numProcessors,
                   final IlluminaDataProviderFactory factory) {

    this.barcodeRecordWriterMap = barcodeRecordWriterMap;
    this.maxReadsInRamPerTile = maxReadsInRamPerTile;
    this.tmpDirs = tmpDirs;
    this.codecPrototype = codecPrototype;
    this.ignoreUnexpectedBarcodes = ignoreUnexpectedBarcodes;
    this.demultiplex = demultiplex;
    this.outputRecordComparator = outputRecordComparator;
    this.bclQualityEvaluationStrategy = bclQualityEvaluationStrategy;
    this.outputRecordClass = outputRecordClass;
    this.factory = factory;


    if (numProcessors == 0) {
        this.numThreads = Runtime.getRuntime().availableProcessors();
    } else if (numProcessors < 0) {
        this.numThreads = Runtime.getRuntime().availableProcessors() + numProcessors;
    } else {
        this.numThreads = numProcessors;
    }
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
void add(final SAMRecord rec) {
    final SortingCollection<SAMRecord> sorter;
    if (outputByReadGroup) {
        sorter = sorterMap.get(rec.getReadGroup().getId());
    } else {
        sorter = singleSorter;
    }
    sorter.add(rec);
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
private SortingCollection<String> makeSortingCollection() {
    final String name = getClass().getSimpleName();
    final File tmpDir = IOUtil.createTempDir(name, null);
    tmpDir.deleteOnExit();
    // 256 byte for one name, and 1/10 part of all memory for this, rough estimate
    long maxNamesInRam = Runtime.getRuntime().maxMemory() / 256 / 10;
    return SortingCollection.newInstance(
            String.class,
            new StringCodec(),
            String::compareTo,
            (int) Math.min(maxNamesInRam, Integer.MAX_VALUE),
            tmpDir
    );
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
@Override
protected int doWork() {
    final List<String> sampleList = new ArrayList<String>();

    for (final File input : INPUT) IOUtil.assertFileIsReadable(input);

    if (SEQUENCE_DICTIONARY != null) IOUtil.assertFileIsReadable(SEQUENCE_DICTIONARY);

    SAMSequenceDictionary samSequenceDictionary = null;
    if (SEQUENCE_DICTIONARY != null) {
        samSequenceDictionary = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).getFileHeader(SEQUENCE_DICTIONARY).getSequenceDictionary();
        CloserUtil.close(SEQUENCE_DICTIONARY);
    }

    // Gather up a file reader and file header for each input file. Check for sequence dictionary compatibility along the way.
    collectFileReadersAndHeaders(sampleList, samSequenceDictionary);

    // Create the merged output header from the input headers
    final VCFHeader outputHeader = new VCFHeader(VCFUtils.smartMergeHeaders(inputHeaders, false), sampleList);

    // Load entries into the sorting collection
    final SortingCollection<VariantContext> sortedOutput = sortInputs(inputReaders, outputHeader);

    // Output to the final file
    writeSortedOutput(outputHeader, sortedOutput);

    return 0;
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
private void writeSortedOutput(final VCFHeader outputHeader, final SortingCollection<VariantContext> sortedOutput) {
    final ProgressLogger writeProgress = new ProgressLogger(log, 25000, "wrote", "records");
    final EnumSet<Options> options = CREATE_INDEX ? EnumSet.of(Options.INDEX_ON_THE_FLY) : EnumSet.noneOf(Options.class);
    final VariantContextWriter out = new VariantContextWriterBuilder().
            setReferenceDictionary(outputHeader.getSequenceDictionary()).
            setOptions(options).
            setOutputFile(OUTPUT).build();
    out.writeHeader(outputHeader);
    for (final VariantContext variantContext : sortedOutput) {
        out.add(variantContext);
        writeProgress.record(variantContext.getContig(), variantContext.getStart());
    }
    out.close();
}
 

import htsjdk.samtools.util.SortingCollection; //导入依赖的package包/类
private void runSingle(final FastqReader r1,final FastqWriter w1) throws IOException
{
long nReads=0;
final  SortingCollection<OneRead> sorting= SortingCollection.newInstance(
		OneRead.class,
		new OneReadCodec(),
		new OneReadCompare(),
		this.writingSortingCollection.getMaxRecordsInRam(),
		this.writingSortingCollection.getTmpPaths()
		);
sorting.setDestructiveIteration(true);
while(r1.hasNext())
	{
	final OneRead r=new OneRead();
	r.random=this.random.nextLong();
	r.index=nReads;
	r.first=r1.next();
	
	if((++nReads)%this.writingSortingCollection.getMaxRecordsInRam()==0)
		{
		LOG.info("Read "+nReads+" reads");
		}

	
	sorting.add(r);
	}
sorting.doneAdding();

final CloseableIterator<OneRead> iter=sorting.iterator();
while(iter.hasNext())
	{
	final OneRead p=iter.next();
	w1.write(p.first);
	}

	
CloserUtil.close(iter);
sorting.cleanup();
}