Python eta.ETA类代码示例

def bam_extract(inbam, outbam, bedfile, nostrand=False, quiet=False):
    bed = BedFile(bedfile)
    if not quiet:
        eta = ETA(os.stat(bedfile).st_size, fileobj=bed)
    else:
        eta = None

    passed = 0

    for region in bed:
        if eta:
            eta.print_status(extra="extracted:%s" % (passed))

        if not region.chrom in inbam.references:
            continue

        if not nostrand:
            strand = region.strand
        else:
            strand = None

        for read in bam_extract_reads(inbam, region.chrom, region.start, region.end, strand):
            outbam.write(read)
            passed += 1

    if not quiet:
        eta.done()
        sys.stderr.write("%s extracted\n" % (passed,))

        def _gen1():
            if not self.quiet:
                eta = ETA(self.regions.total)
            else:
                eta = None

            count = 0
            for region in self.regions:
                working_chrom = None
                if region.chrom in self.bam.references:
                    working_chrom = region.chrom
                elif chrom[0:3] == 'chr':
                        if region.chrom[3:] in self.bam.references:
                            working_chrom = region.chrom[3:]

                if not working_chrom:
                    continue

                # for troubleshooting
                self.cur_chrom = region.chrom
                self.cur_start = region.start
                self.cur_end = region.end

                laststart = 0
                for read in self.bam.fetch(working_chrom, region.start, region.end):
                    if read.pos != laststart:
                        count += 1
                        laststart = read.pos

                    if eta:
                        eta.print_status(count, extra='%s/%s %s:%s' % (count, self.regions.total, self.bam.references[read.tid], read.pos))

                    yield read
            if eta:
                eta.done()

def gzip_reader(fname, quiet=False, callback=None, done_callback=None):
    if fname == '-':
        f = sys.stdin
    elif fname[-3:] == '.gz' or fname[-4:] == '.bgz':
        f = gzip.open(os.path.expanduser(fname))
    else:
        f = open(os.path.expanduser(fname))

    if quiet or fname == '-':
        eta = None
    else:
        eta = ETA(os.stat(fname).st_size, fileobj=f)

    for line in f:
        if eta:
            if callback:
                extra = callback()
            else:
                extra = ''

            eta.print_status(extra=extra)
        yield line

        if done_callback and done_callback():
                break

    if f != sys.stdout:
        f.close()

    if eta:
        eta.done()

 def _schedualJobs(self,jobs):
     ''' strategy for jobs
     '''
     time_interval = self.time_interval
     with open(self.log_file,'a') as logFile:
         n_err = 0
         eta = ETA(len(jobs))
         for i in xrange(len(jobs)):
             eta.print_status(i)
             time1 = time.time()
             result =self._doJob(jobs[i],logFile) 
             if(result != None  ):
                 self._saveLog(logFile,jobs[i],success=True)
                 self._collect(result)
                 if self.result_file!=None :
                     self._saveResult()  
                 time_interval = time_interval/4
                 if time_interval < self.time_interval:
                     time_interval = self.time_interval               
             else:
                 self._saveLog(logFile,jobs[i],success=False)
                 n_err += 1
                 time_interval = time_interval*2
             if n_err == self.error_times:
                 break;
             time_span = time.time() - time1
             if(time_span<self.time_interval):
                 time.sleep(self.time_interval - time_span)
         eta.done()

def fastq_sort(fastq, byname=True, bysequence=False, tmpdir=None, chunksize=100000, out=sys.stdout, quiet=False):
    tmpfiles = []

    chunk = []
    sys.stderr.write('Sorting FASTQ file into chunks...\n')
    count = 0
    for read in fastq.fetch(quiet):
        count += 1 
        if byname:
            chunk.append((read.name, read))
        if bysequence:
            chunk.append((read.seq, read))

        if len(chunk) >= chunksize:
            tmpfiles.append(_write_tmp(chunk))
            chunk = []

    if chunk:
        tmpfiles.append(_write_tmp(chunk))

    sys.stderr.write('Merging chunks...\n')
    buf = [None, ] * len(tmpfiles)
    skip = [False, ] * len(tmpfiles)

    eta = ETA(count)

    j=0
    writing = True

    while writing:
        j+=1
        eta.print_status(j)
        for i, fobj in enumerate(tmpfiles):
            if not buf[i] and not skip[i]:
                try:
                    read = fastq_read_file(fobj)
                    if byname:
                        buf[i] = (read.name, i, read)
                    if bysequence:
                        buf[i] = (read.seq, i, read)
                except:
                    buf[i] = None
                    skip[i] = True
        
        sorted_list = buf[:]
        sorted_list.sort()
        writing = False

        for tup in sorted_list:
            if not tup:
                continue

            sorter, i, read = tup
            read.write(out)
            buf[i] = None
            writing = True
            break
    eta.done()

def gtf_junctions(gtf, refname, fragment_size, min_size, max_exons=5, known=False, out=sys.stdout, quiet=False, scramble=False, retain_introns=False):
    ref = pysam.Fastafile(refname)

    references = []
    with open('%s.fai' % refname) as f:
        for line in f:
            cols = line.split('\t')
            references.append(cols[0])

    if not quiet:
        eta = ETA(gtf.fsize(), fileobj=gtf)
    else:
        eta = None

    exporter = JunctionExporter(ref, fragment_size, min_size, max_exons, out, scramble)

    for gene in gtf.genes:
        if not gene.chrom in references:
            continue

        if eta:
            eta.print_status(extra='%s:%s %s' % (gene.chrom, gene.start, gene.gene_name))

        if known:
            for txpt in gene.transcripts:
                last = None
                for exon in txpt.exons:
                    if last:
                        exporter.export(gene.chrom, [last, exon])
                    last = exon
        else:
            exons = set()
            for txpt in gene.transcripts:
                for exon in txpt.exons:
                    exons.add(exon)

            exons = list(exons)
            exons.sort()

            if retain_introns:
                exporter.export_retained_introns(gene.chrom, exons, gene.strand)

            if scramble:
                # We can just pretend the transcript is repeated
                # and then let the set take care of removing the duplicates
                exons = exons * 2

            exporter.export(gene.chrom, exons)



    if eta:
        eta.done()
    ref.close()

class Callback(object):
    def __init__(self, total):
        self.i = 0
        self.eta = ETA(total)

    def __call__(self, result=None):
        self.i += 1
        self.eta.print_status(self.i, extra=result if result else '')

    def done(self):
        self.eta.done()

 def output(self, partitions):
     for pn, part in enumerate(partitions):
         d = os.path.abspath(os.path.join(self.dst, self.name % pn))
         if os.path.isfile(d):
             logging.warning('Archive already exists, overwriting: ' + d)
         logging.info('Creating archive %s...' % (self.name % pn))
         eta = ETA(part.size, min_ms_between_updates=500)
         with tarfile.open(d, self.mode) as tar:
             for fn, size, estsize in part.filelist:
                 try:
                     tar.add(os.path.join(self.srcbase, fn), fn)
                 except Exception as ex:
                     logging.error(ex)
                 eta.print_status(estsize)
         eta.done()

def gzip_aware_reader(fname, callback=None):
    if fname[-3:] == '.gz':
        f = gzip.open(fname)
    else:
        f = open(fname)

    eta = ETA(os.stat(fname).st_size, fileobj=f)
    for line in f:
        extra = None
        if callback:
            extra = callback()
        eta.print_status(extra=extra)
        yield line

    f.close()
    eta.done()

    def fetch(self, quiet=False):
        if self.fname and not quiet:
            eta = ETA(os.stat(self.fname).st_size, fileobj=self.fileobj)
        else:
            eta = None

        while True:
            try:
                read = fastq_read_file(self.fileobj)
                if eta:
                    eta.print_status(extra=read.name)
                yield read

            except:
                break

        if eta:
            eta.done()

def _repeatreader(fname):
    with ngsutils.support.ngs_utils.gzip_opener(fname) as repeat_f:
        eta = ETA(os.stat(fname).st_size, fileobj=repeat_f)
        repeat_f.next()
        repeat_f.next()
        repeat_f.next()

        for line in repeat_f:
            cols = line.strip().split()
            chrom = cols[4]
            start = int(cols[5]) - 1
            end = int(cols[6])
            strand = '+' if cols[8] == '+' else '-'
            family = cols[10]
            member = cols[9]

            eta.print_status(extra='%s|%s %s:%s-%s[%s]' % (family, member, chrom, start, end, strand))
            yield (family, member, chrom, start, end, strand)
        eta.done()

def gtf_junctions(gtf, refname, fragment_size, min_size, max_exons=5, known=False, out=sys.stdout, quiet=False):
    ref = pysam.Fastafile(refname)

    references = []
    with open("%s.fai" % refname) as f:
        for line in f:
            cols = line.split("\t")
            references.append(cols[0])

    if not quiet:
        eta = ETA(gtf.fsize(), fileobj=gtf)
    else:
        eta = None

    exporter = JunctionExporter(ref, fragment_size, min_size, max_exons, out)

    for gene in gtf.genes:
        if not gene.chrom in references:
            continue

        if eta:
            eta.print_status(extra="%s:%s %s" % (gene.chrom, gene.start, gene.gene_name))

        if known:
            for txpt in gene.transcripts:
                last = None
                for exon in txpt.exons:
                    if last:
                        exporter.export(gene.chrom, [last, exon])
                    last = exon
        else:
            exons = set()
            for txpt in gene.transcripts:
                for exon in txpt.exons:
                    exons.add(exon)
            exons = list(exons)
            exons.sort()
            exporter.export(gene.chrom, exons)

    if eta:
        eta.done()
    ref.close()

    def get_regions(self):
        eta = ETA(self.gtf.fsize(), fileobj=self.gtf)

        for gene in self.gtf.genes:
            eta.print_status(extra=gene.gene_name)
            starts = []
            ends = []

            # just include all regions - don't worry about transcripts and exons
            # the regions encompass all exons anyway...
            for num, start, end, const, names in gene.regions:
                starts.append(start)
                ends.append(end)

            out = [gene.gene_id, gene.gene_name, ]
            if self.has_isoform:
                out.append(gene.attributes['isoform_id'] if 'isoform_id' in gene.attributes else '')
            if self.has_biotype:
                out.append(gene.attributes['gene_biotype'] if 'gene_biotype' in gene.attributes else '')
            out.extend([gene.chrom, gene.strand, gene.start, gene.end])

            yield (gene.chrom, starts, ends, gene.strand, out, None)
        eta.done()

def run(predict, test_size, n_iter=100, n_burnin=10, resample=None):
    p_tot = LogR(0)
    eta = ETA(n_iter); eta.print_status(0, extra='starting...')
    for i in range(n_iter):
        print polya.timestamp(), "iteration %u/%u" % (i+1, n_iter)
        polya.resample()
        if resample: resample()
        
        p = predict()
        pplx = float(p ** (-1./test_size))
        print polya.timestamp(), 'perplexity =', pplx
        if i < n_burnin:
            eta.print_status(i+1, extra="burning in (%.1f)..." % pplx)
        else:
            p_tot += p
            pplx = float((p_tot / (i+1 - n_burnin)) ** (-1./test_size))
            eta.print_status(i+1, extra="perplexity %.1f" % pplx)
    eta.done()
    p_avg = p_tot / (n_iter - n_burnin)
    pplx = float(p_avg ** (-1./test_size))
    print '---\nfinal perplexity =', pplx
    print>>sys.stderr, 'Perplexity:', pplx
    return p_avg

    def fetch(self, quiet=False):
        name = ''
        comment = ''
        seq = ''

        if not quiet and self.fname and self.fname != '-':
            eta = ETA(os.stat(self.fname).st_size, fileobj=self.fileobj)
        else:
            eta = None

        for line in self.fileobj:
            line = line.strip()
            if not line:
                continue
            if line[0] == '#':
                continue

            if line[0] == '>':
                if name and seq:
                    if eta:
                        eta.print_status(extra=name)
                    yield FASTARead(name, comment, seq)

                spl = re.split(r'[ \t]', line[1:], maxsplit=1)
                name = spl[0]
                if len(spl) > 1:
                    comment = spl[1]
                else:
                    comment = ''
                seq = ''

            else:
                if self.qual:
                    seq = seq + ' ' + line
                else:
                    seq += line

        if name and seq:
            if eta:
                eta.print_status(extra=name)
            yield FASTARead(name, comment, seq)

        if eta:
            eta.done()