Python ETA.done方法代码示例

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def gzip_reader(fname, quiet=False, callback=None, done_callback=None):
    if fname == '-':
        f = sys.stdin
    elif fname[-3:] == '.gz' or fname[-4:] == '.bgz':
        f = gzip.open(os.path.expanduser(fname))
    else:
        f = open(os.path.expanduser(fname))

    if quiet or fname == '-':
        eta = None
    else:
        eta = ETA(os.stat(fname).st_size, fileobj=f)

    for line in f:
        if eta:
            if callback:
                extra = callback()
            else:
                extra = ''

            eta.print_status(extra=extra)
        yield line

        if done_callback and done_callback():
                break

    if f != sys.stdout:
        f.close()

    if eta:
        eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
 def _schedualJobs(self,jobs):
     ''' strategy for jobs
     '''
     time_interval = self.time_interval
     with open(self.log_file,'a') as logFile:
         n_err = 0
         eta = ETA(len(jobs))
         for i in xrange(len(jobs)):
             eta.print_status(i)
             time1 = time.time()
             result =self._doJob(jobs[i],logFile) 
             if(result != None  ):
                 self._saveLog(logFile,jobs[i],success=True)
                 self._collect(result)
                 if self.result_file!=None :
                     self._saveResult()  
                 time_interval = time_interval/4
                 if time_interval < self.time_interval:
                     time_interval = self.time_interval               
             else:
                 self._saveLog(logFile,jobs[i],success=False)
                 n_err += 1
                 time_interval = time_interval*2
             if n_err == self.error_times:
                 break;
             time_span = time.time() - time1
             if(time_span<self.time_interval):
                 time.sleep(self.time_interval - time_span)
         eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def bam_iter(bam, quiet=False, show_ref_pos=False, callback=None):
    '''
    >>> [x.qname for x in bam_iter(bam_open(os.path.join(os.path.dirname(__file__), 't', 'test.bam')), quiet=True)]
    ['A', 'B', 'E', 'C', 'D', 'F', 'Z']
    '''
    if not quiet and bam.filename:
        eta = ETA(os.stat(bam.filename).st_size)
    else:
        eta = None

    if os.path.exists('%s.bai' % bam.filename):
        # This is an indexed file, so it is ref sorted...
        # Meaning that we should show chrom:pos, instead of read names
        show_ref_pos = True

    for read in bam:
        pos = bam.tell()
        bgz_offset = pos >> 16

        if not quiet and eta:
            if callback:
                eta.print_status(bgz_offset, extra=callback(read))
            elif (show_ref_pos):
                if read.tid > -1:
                    eta.print_status(bgz_offset, extra='%s:%s %s' % (bam.getrname(read.tid), read.pos, read.qname))
                else:
                    eta.print_status(bgz_offset, extra='unmapped %s' % (read.qname))
            else:
                eta.print_status(bgz_offset, extra='%s' % read.qname)
        yield read

    if eta:
        eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def bam_extract(inbam, outbam, bedfile, nostrand=False, quiet=False):
    bed = BedFile(bedfile)
    if not quiet:
        eta = ETA(os.stat(bedfile).st_size, fileobj=bed)
    else:
        eta = None

    passed = 0

    for region in bed:
        if eta:
            eta.print_status(extra="extracted:%s" % (passed))

        if not region.chrom in inbam.references:
            continue

        if not nostrand:
            strand = region.strand
        else:
            strand = None

        for read in bam_extract_reads(inbam, region.chrom, region.start, region.end, strand):
            outbam.write(read)
            passed += 1

    if not quiet:
        eta.done()
        sys.stderr.write("%s extracted\n" % (passed,))

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
        def _gen1():
            if not self.quiet:
                eta = ETA(self.regions.total)
            else:
                eta = None

            count = 0
            for region in self.regions:
                working_chrom = None
                if region.chrom in self.bam.references:
                    working_chrom = region.chrom
                elif chrom[0:3] == 'chr':
                        if region.chrom[3:] in self.bam.references:
                            working_chrom = region.chrom[3:]

                if not working_chrom:
                    continue

                # for troubleshooting
                self.cur_chrom = region.chrom
                self.cur_start = region.start
                self.cur_end = region.end

                laststart = 0
                for read in self.bam.fetch(working_chrom, region.start, region.end):
                    if read.pos != laststart:
                        count += 1
                        laststart = read.pos

                    if eta:
                        eta.print_status(count, extra='%s/%s %s:%s' % (count, self.regions.total, self.bam.references[read.tid], read.pos))

                    yield read
            if eta:
                eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
    def get_regions(self):
        total = 0
        for chrom, chrom_len in self.chrom_lens:
            total += (chrom_len / self.binsize)
            if chrom_len % self.binsize != 0:
                total += 1

        eta = ETA(total)
        pos_acc = 0
        for chrom, chrom_len in self.chrom_lens:
            pos = -1
            for bin in xrange(0, chrom_len, self.binsize):
                if pos > -1:
                    eta.print_status(pos_acc, extra='%s:%s[+]' % (chrom, bin))
                    yield (chrom, [pos], [bin], '+', [chrom, pos, bin, '+'], None)
                    if self.stranded:
                        eta.print_status(pos_acc, extra='%s:%s[-]' % (chrom, bin))
                        yield (chrom, [pos], [bin], '-', [chrom, pos, bin, '-'], None)
                pos = bin
                pos_acc += 1

            eta.print_status(pos_acc, extra='%s:%s[+]' % (chrom, bin))
            yield (chrom, [pos], [chrom_len], '+', [chrom, pos, chrom_len, '+'], None)
            if self.stranded:
                eta.print_status(pos_acc, extra='%s:%s[-]' % (chrom, bin))
                yield (chrom, [pos], [chrom_len], '- ', [chrom, pos, chrom_len, '-'], None)

        eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def fastq_sort(fastq, byname=True, bysequence=False, tmpdir=None, chunksize=100000, out=sys.stdout, quiet=False):
    tmpfiles = []

    chunk = []
    sys.stderr.write('Sorting FASTQ file into chunks...\n')
    count = 0
    for read in fastq.fetch(quiet):
        count += 1 
        if byname:
            chunk.append((read.name, read))
        if bysequence:
            chunk.append((read.seq, read))

        if len(chunk) >= chunksize:
            tmpfiles.append(_write_tmp(chunk))
            chunk = []

    if chunk:
        tmpfiles.append(_write_tmp(chunk))

    sys.stderr.write('Merging chunks...\n')
    buf = [None, ] * len(tmpfiles)
    skip = [False, ] * len(tmpfiles)

    eta = ETA(count)

    j=0
    writing = True

    while writing:
        j+=1
        eta.print_status(j)
        for i, fobj in enumerate(tmpfiles):
            if not buf[i] and not skip[i]:
                try:
                    read = fastq_read_file(fobj)
                    if byname:
                        buf[i] = (read.name, i, read)
                    if bysequence:
                        buf[i] = (read.seq, i, read)
                except:
                    buf[i] = None
                    skip[i] = True
        
        sorted_list = buf[:]
        sorted_list.sort()
        writing = False

        for tup in sorted_list:
            if not tup:
                continue

            sorter, i, read = tup
            read.write(out)
            buf[i] = None
            writing = True
            break
    eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def gtf_junctions(gtf, refname, fragment_size, min_size, max_exons=5, known=False, out=sys.stdout, quiet=False, scramble=False, retain_introns=False):
    ref = pysam.Fastafile(refname)

    references = []
    with open('%s.fai' % refname) as f:
        for line in f:
            cols = line.split('\t')
            references.append(cols[0])

    if not quiet:
        eta = ETA(gtf.fsize(), fileobj=gtf)
    else:
        eta = None

    exporter = JunctionExporter(ref, fragment_size, min_size, max_exons, out, scramble)

    for gene in gtf.genes:
        if not gene.chrom in references:
            continue

        if eta:
            eta.print_status(extra='%s:%s %s' % (gene.chrom, gene.start, gene.gene_name))

        if known:
            for txpt in gene.transcripts:
                last = None
                for exon in txpt.exons:
                    if last:
                        exporter.export(gene.chrom, [last, exon])
                    last = exon
        else:
            exons = set()
            for txpt in gene.transcripts:
                for exon in txpt.exons:
                    exons.add(exon)

            exons = list(exons)
            exons.sort()

            if retain_introns:
                exporter.export_retained_introns(gene.chrom, exons, gene.strand)

            if scramble:
                # We can just pretend the transcript is repeated
                # and then let the set take care of removing the duplicates
                exons = exons * 2

            exporter.export(gene.chrom, exons)



    if eta:
        eta.done()
    ref.close()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
class Callback(object):
    def __init__(self, total):
        self.i = 0
        self.eta = ETA(total)

    def __call__(self, result=None):
        self.i += 1
        self.eta.print_status(self.i, extra=result if result else '')

    def done(self):
        self.eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
 def output(self, partitions):
     for pn, part in enumerate(partitions):
         d = os.path.abspath(os.path.join(self.dst, self.name % pn))
         if os.path.isfile(d):
             logging.warning('Archive already exists, overwriting: ' + d)
         logging.info('Creating archive %s...' % (self.name % pn))
         eta = ETA(part.size, min_ms_between_updates=500)
         with zipfile.ZipFile(d, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
             for fn, size, estsize in part.filelist:
                 try:
                     zipf.write(os.path.join(self.srcbase, fn), fn)
                 except Exception as ex:
                     logging.error(ex)
                 eta.print_status(estsize)
         eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def gzip_aware_reader(fname, callback=None):
    if fname[-3:] == '.gz':
        f = gzip.open(fname)
    else:
        f = open(fname)

    eta = ETA(os.stat(fname).st_size, fileobj=f)
    for line in f:
        extra = None
        if callback:
            extra = callback()
        eta.print_status(extra=extra)
        yield line

    f.close()
    eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
    def fetch(self, quiet=False):
        if self.fname and not quiet:
            eta = ETA(os.stat(self.fname).st_size, fileobj=self.fileobj)
        else:
            eta = None

        while True:
            try:
                read = fastq_read_file(self.fileobj)
                if eta:
                    eta.print_status(extra=read.name)
                yield read

            except:
                break

        if eta:
            eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
    def fetch(self, quiet=False):
        name = ''
        comment = ''
        seq = ''

        if not quiet and self.fname and self.fname != '-':
            eta = ETA(os.stat(self.fname).st_size, fileobj=self.fileobj)
        else:
            eta = None

        for line in self.fileobj:
            line = line.strip()
            if not line:
                continue
            if line[0] == '#':
                continue

            if line[0] == '>':
                if name and seq:
                    if eta:
                        eta.print_status(extra=name)
                    yield FASTARead(name, comment, seq)

                spl = re.split(r'[ \t]', line[1:], maxsplit=1)
                name = spl[0]
                if len(spl) > 1:
                    comment = spl[1]
                else:
                    comment = ''
                seq = ''

            else:
                if self.qual:
                    seq = seq + ' ' + line
                else:
                    seq += line

        if name and seq:
            if eta:
                eta.print_status(extra=name)
            yield FASTARead(name, comment, seq)

        if eta:
            eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def _repeatreader(fname):
    with ngsutils.support.ngs_utils.gzip_opener(fname) as repeat_f:
        eta = ETA(os.stat(fname).st_size, fileobj=repeat_f)
        repeat_f.next()
        repeat_f.next()
        repeat_f.next()

        for line in repeat_f:
            cols = line.strip().split()
            chrom = cols[4]
            start = int(cols[5]) - 1
            end = int(cols[6])
            strand = '+' if cols[8] == '+' else '-'
            family = cols[10]
            member = cols[9]

            eta.print_status(extra='%s|%s %s:%s-%s[%s]' % (family, member, chrom, start, end, strand))
            yield (family, member, chrom, start, end, strand)
        eta.done()

# 需要导入模块: from eta import ETA [as 别名]
# 或者: from eta.ETA import done [as 别名]
def gtf_junctions(gtf, refname, fragment_size, min_size, max_exons=5, known=False, out=sys.stdout, quiet=False):
    ref = pysam.Fastafile(refname)

    references = []
    with open("%s.fai" % refname) as f:
        for line in f:
            cols = line.split("\t")
            references.append(cols[0])

    if not quiet:
        eta = ETA(gtf.fsize(), fileobj=gtf)
    else:
        eta = None

    exporter = JunctionExporter(ref, fragment_size, min_size, max_exons, out)

    for gene in gtf.genes:
        if not gene.chrom in references:
            continue

        if eta:
            eta.print_status(extra="%s:%s %s" % (gene.chrom, gene.start, gene.gene_name))

        if known:
            for txpt in gene.transcripts:
                last = None
                for exon in txpt.exons:
                    if last:
                        exporter.export(gene.chrom, [last, exon])
                    last = exon
        else:
            exons = set()
            for txpt in gene.transcripts:
                for exon in txpt.exons:
                    exons.add(exon)
            exons = list(exons)
            exons.sort()
            exporter.export(gene.chrom, exons)

    if eta:
        eta.done()
    ref.close()