Python ProgressBar.ProgressBar类代码示例

	def processFiles(self, files, folderPath):
		"""function processFiles
		
		returns []
		"""
		if _logger.getEffectiveLevel() == 30:
			import os
			sumSize			= 0
			for file in files:
				sumSize		+= os.path.getsize(folderPath+file)
			bar				= ProgressBar(sumSize)
		else:
			bar				= False

		#Creating MULTI FASTA file, with all genens
		import os
		fastaFile = open(os.path.expanduser("~/Documents/th6_MaartenJoost_multi.fasta"), 'w')

		# evt [http://docs.python.org/2/library/multiprocessing.html]
		for file in sorted(files):
			self.chromosomeNumber	+= 1
			chromosome				 = Chromosome(file, folderPath, _DB, _logger, fastaFile)
			self.oligosNumber		+= chromosome.oligosNumber
			self.genesNumber		+= chromosome.genesNum
			del(chromosome)
			if bar: bar.update(os.path.getsize(folderPath+file))

		if bar: del(bar)

    def ProduceSiteReadinessSSBFile(self):
        print "\nProducing Site Readiness SSB input file\n"
        prog = ProgressBar(0, 100, 77)
        fileHandle = open(self.fileSSB, "w")

        sitesit = self.matrices.readiValues.keys()
        sitesit.sort()
        for sitename in sitesit:
            prog.increment(100.0 / len(sitesit))

            if self.SkipSiteOutput(sitename):
                continue

            status = self.matrices.readiValues[sitename][self.tinfo.yesterdaystamp]
            colorst = self.SRMatrixColors[status]

            linkSSB = self.options.url + "/SiteReadiness/HTML/SiteReadinessReport_" + self.tinfo.timestamphtml + ".html"
            tofile = (
                self.tinfo.todaystampfileSSB
                + "\t"
                + sitename
                + "\t"
                + status
                + "\t"
                + colorst
                + "\t"
                + linkSSB
                + "#"
                + sitename
                + "\n"
            )
            fileHandle.write(tofile)

        fileHandle.close()
        prog.finish()

    def ProduceSiteReadinessSSBFiles(self):
        print "\nProducing Site Readiness SSB files to commission view\n"
        prog = ProgressBar(0, 100, 77)
        for dayspan in 30, 15, 7:
            prog.increment(100./3.)
            fileSSBRanking = self.ssbOutDir + '/SiteReadinessRanking_SSBfeed_last' + str(dayspan) + 'days.txt'
            fileHandle = open ( fileSSBRanking , 'w' )

            sitesit = self.matrices.readiValues.keys()
            sitesit.sort()
            for sitename in sitesit:
                if self.SkipSiteOutput(sitename): continue
                pl = "R+Wcorr_perc"
                color = "red"
                if sitename.find("T1") == 0 and self.matrices.stats[sitename][dayspan][pl]>90:
                    color="green"
                if sitename.find("T2") == 0 and self.matrices.stats[sitename][dayspan][pl]>80:
                    color="green"
                if self.matrices.stats[sitename][dayspan][pl] != "n/a":
                    filenameSSB = self.options.url + "/SiteReadiness/PLOTS/" + sitename.split("_")[0] + "_" + pl + "_last" + str(dayspan) + "days_" + self.tinfo.timestamphtml + ".png"
                    tofile = self.tinfo.todaystampfileSSB + '\t' + sitename + '\t' + str(self.matrices.stats[sitename][dayspan][pl]) + '\t' + color + '\t' + filenameSSB + "\n"
                    fileHandle.write(tofile)

        fileHandle.close()
        prog.finish()

    def PrintDailyMetrics(self):
        prog = ProgressBar(0, 100, 77)
        indmetrics = self.cinfo.metorder.keys()
        indmetrics.sort()
        sites = self.matrices.columnValues.keys()
        sites.sort()
        for sitename in sites:
            prog.increment(100./3.)
            dates = self.matrices.dailyMetrics[sitename].keys()
            dates.sort()
            for dat in dates:
                if self.SkipSiteOutput(sitename): continue
                for metnumber in indmetrics:
                    met = self.cinfo.metorder[metnumber] #colName
                    met1 = self.cinfo.printCol[met] #pCol (print permission)

                    #if not self.matrices.columnValues[sitename][dat].has_key(met) or met == 'IsSiteInSiteDB': continue # ignore
                    if not self.matrices.columnValues[sitename][dat].has_key(met) or met1 == '0' : continue # ignore

                    if self.matrices.columnValues[sitename][dat][met].has_key('URL'):
                        url = self.matrices.columnValues[sitename][dat][met]['URL']
                    else:
                        url = "-"
                    print dat, sitename, met, self.matrices.columnValues[sitename][dat][met]['Status'], self.matrices.columnValues[sitename][dat][met]['Color'],url

        prog.finish()

    def PrintDailyMetricsStats(self):
        print "\nPrinting Daily Metrics Statistics\n"
        prog = ProgressBar(0, 100, 77)
        fileHandle = open(self.asciiOutDir + "/Daily_HistoricStatistics.txt", "w")
        sites = self.matrices.dailyMetrics.keys()
        sites.sort()
        for sitename in sites:
            dates = self.matrices.dailyMetrics[sitename].keys()
            dates.sort()
            continue

        for i in "T1", "T2":
            prog.increment(100.0 / 2.0)
            for dat in dates:
                countO = 0
                countE = 0
                countSD = 0
                countna = 0
                for sitename in sites:
                    if sitename.find("T1_CH_CERN") == 0:
                        continue
                    if not sitename.find(i + "_") == 0:
                        continue
                    if self.SkipSiteOutput(sitename):
                        continue

                    state = self.matrices.dailyMetrics[sitename][dat]
                    if state == "O":
                        countO += 1
                    if state == "E":
                        countE += 1
                    if state.find("n/a") == 0:
                        countna += 1
                    if state == "SD":
                        countSD += 1

                if dat == self.tinfo.todaystamp:
                    continue
                tofile = (
                    "Daily Metric "
                    + i
                    + " "
                    + dat
                    + " "
                    + str(countE)
                    + " "
                    + str(countO)
                    + " "
                    + str(countna)
                    + " "
                    + str(countSD)
                    + " "
                    + str(countE + countO + countSD + countna)
                    + "\n"
                )
                fileHandle.write(tofile)

        fileHandle.close()
        prog.finish()

    def ParseXML(self):
        print "\nObtaining XML info from SSB 'Site Readiness' view\n"
        prog = ProgressBar(0, 100, 77)

        xmlCacheDir = self.options.path_out + "/INPUTxmls"
        if not os.path.exists(xmlCacheDir):
            os.makedirs(xmlCacheDir)
        ColumnItems = self.cinfo.urls.keys()
        ColumnItems.sort()
        for col in ColumnItems:
            prog.increment(100./len(ColumnItems))
            url = self.cinfo.urls[col]
            xmlFile = xmlCacheDir + "/" + col + ".xml"
            if self.options.xml == 'false' and not os.path.exists(xmlCacheDir):
                print "\nWARNING: you cannot re-use the XML files as the files were not obtained before. Obtaining them...\n"
                self.options.xml = 'true'
            if self.options.xml == 'true': # download xml file if requested
                print "Column %s - Getting the url %s" % (col, url)
                os.system("curl -s -H 'Accept: text/xml'  '%s' > %s" % (url,xmlFile))
        
            f = file(xmlFile,'r') # read xml file that was either just written, or was written in the previous run
            t = xml.dom.minidom.parse(f)
            f.close()
            #print t.toprettyxml() 
            for subUrl in xpath.Evaluate("/getplotdata/csvdata/item", t):
                #print subUrl.toprettyxml()
                info = {} # basic info about the site for this column
                for option in ('Status', "COLOR", 'Time', 'EndTime','VOName','URL'):
                    for target in xpath.Evaluate(option, subUrl):
                        if target.hasChildNodes():
                            s = target.firstChild.nodeValue.encode('ascii')
                        else:
                            s = ""
                        info[option] = s

                voname = info['VOName']
                time = info['Time']
                xmlMatrix = self.matrices.xmlInfo
                if self.options.oneSite != "" and voname.find(self.options.oneSite) != 0: continue
                if not xmlMatrix.has_key(voname): # if site not already in dict, add an empty dict for it
                    xmlMatrix[voname] = {}
                if not xmlMatrix[voname].has_key(col): # if site entry doesn't already have this column, add an empty dict for this column
                    xmlMatrix[voname][col] = {}
                xmlMatrix[voname][col][time] = info # set the actual values

                # Correct some of the strings
                value = xmlMatrix[voname][col][time]['Status']
                if col=="HammerCloud" and value != "n/a":
                    value = str(int(float(value)))
                    if value.find("%") != 0:
                        value += "%"
                elif col=="SUMAvailability":
                    value = str(int(round(float(value))))
                    if value.find("%") != 0:
                        value += "%"
                xmlMatrix[voname][col][time]['Status'] = value
    
        prog.finish()

def main( iterations, ensembles, episodes, agent_type, agent_args, env_type, env_args, file_prefix ):
    """RL Testbed.
    @arg iterations: Number of environments to average over
    @arg ensembles: Number of bots to average over
    @arg episodes: Number of episodes to run for
    @arg agent_type: String name of agent
    @arg agent_args: Arguments to the agent constructor
    @arg env_type: String name of environment
    @arg env_args: Arguments to the environment constructor
    """
    # Load agent and environment

    progress = ProgressBar( 0, ensembles*iterations, mode='fixed' )
    # Needed to prevent glitches
    oldprog = str(progress)

    # Counters
    ret = np.zeros( episodes, dtype=float )
    min_, max_ = np.inf * np.ones( episodes, dtype=float) , -np.inf * np.ones( episodes, dtype=float)
    var = np.zeros( episodes, dtype=float )

    env = env_type.create( *env_args )
    for i in xrange( 1, iterations+1 ):
        env = env.domain.reset_rewards( env, *env_args )

        ret_ = np.zeros( episodes, dtype=float )
        # Initialise environment and agent
        for j in xrange( 1, ensembles+1 ):
            print "iter, ens", i, j
            agent = agent_type( env.Q, *agent_args )
            ret__ = Runner.run( env, agent, episodes )
            ret__ = np.cumsum( ret__ ) #Varun
            # Add to ret_
            ret_ += (ret__ - ret_) / j

            # print progress
            progress.increment_amount()
            if oldprog != str(progress):
                print progress, "\r",
                sys.stdout.flush()
                oldprog=str(progress)

        ret += (ret_ - ret) / i
        min_ = np.min( np.vstack( ( min_, ret_ ) ), axis=0 )
        max_ = np.max( np.vstack( ( max_, ret_ ) ), axis=0 )

        var_ = np.power( ret_, 2 )
        var += (var_ - var) / i
    print "\n"

    var = np.sqrt( var - np.power( ret, 2 ) )

    f = open("%s-return.dat"%( file_prefix ), "w")
    # Print ret
    for i in xrange( len( ret ) ):
        f.write( "%d %f %f %f %f\n"%( i+1, ret[ i ], min_[i], max_[i], var[ i ] ) )
    f.close()

    def ProduceSiteReadinessStatistics(self):
        print "\nProducing Site Readiness Statistics\n"
        sitesit = self.matrices.readiValues.keys()
        sitesit.sort()
        prog = ProgressBar(0, 100, 77)
        for dayspan in 30, 15, 7:
            prog.increment(100./3.)
            for sitename in sitesit:
                if self.SkipSiteOutput(sitename): continue

                countR = 0;  countW = 0;  countNR = 0;  countSD = 0;  countNA = 0
                infostats2 = {}
                if not self.matrices.stats.has_key(sitename):
                    self.matrices.stats[sitename]={}

                for i in range(0,dayspan):
                    deltaT = datetime.timedelta(i)
                    datestamp = self.tinfo.yesterday - deltaT

                    state = self.matrices.readiValues[sitename][datestamp.strftime("%Y-%m-%d")]

                    if state == "R":  countR  += 1
                    if state == "W":  countW  += 1
                    if state == "NR": countNR += 1
                    if state == "SD": countSD += 1
                    if state.find("n/a") == 0: countNA += 1

                if not self.matrices.stats[sitename].has_key(dayspan):
                    self.matrices.stats[sitename][dayspan]={}

                infostats2['R_perc']= (int)(round(100.*countR/dayspan))
                infostats2['W_perc']= (int)(round(100.*countW/dayspan))
                infostats2['R+W_perc']= (int)(round(100.*(countR+countW)/dayspan))
                infostats2['NR_perc']= (int)(round(100.*countNR/dayspan))
                infostats2['SD_perc']= (int)(round(100.*countSD/dayspan))
                infostats2['R']= countR
                infostats2['W']= countW
                infostats2['R+W']= countW+countR
                infostats2['NR']= countNR
                infostats2['SD']= countSD
                infostats2['days']=dayspan

                if (dayspan-countSD-countNA)!=0:
                    infostats2['Rcorr_perc']= (int)(round(100.*countR/(dayspan-countSD-countNA)))
                    infostats2['Wcorr_perc']= (int)(round(100.*countW/(dayspan-countSD-countNA)))
                    infostats2['R+Wcorr_perc']= (int)(round(100.*(countR+countW)/(dayspan-countSD-countNA)))
                    infostats2['NRcorr_perc']= (int)(round(100.*countNR/(dayspan-countSD-countNA)))
                else:
                    infostats2['Rcorr_perc']= 0
                    infostats2['Wcorr_perc']= 0
                    infostats2['R+Wcorr_perc']= 0
                    infostats2['NRcorr_perc']= 100

                self.matrices.stats[sitename][dayspan]=infostats2

        prog.finish()

class RawDataManager(QtGui.QWidget):
    def __init__(self, user_id, password, *args, **kwargs):
        super(RawDataManager, self).__init__(*args, **kwargs)
        self.user_id, self.password = user_id, password
        self.raw_data_thread = RawDataUploaderThread(self.user_id, self.password)
        self.createUI()
        self.mapEvents()

    def createUI(self):
        #self.raw_data_filter_form = FilterForm()
        self.fsn_entry_field = FSNTextEdit()
        self.raw_data_table = CopiableQTableWidget(0,0)
        self.upload_raw_data_button = QtGui.QPushButton("Upload Raw Data from File")
        self.progress_bar = ProgressBar()
        self.progress_log = QtGui.QTextEdit()
        layout = QtGui.QVBoxLayout()
        layout.addWidget(self.upload_raw_data_button)
        layout.addWidget(self.progress_bar)
        layout.addWidget(self.progress_log)
        self.setLayout(layout)
        self.setWindowTitle("Raw Data Uploader")
        self.setWindowIcon(QtGui.QIcon(os.path.join(MOSES.getPathToImages(),"PORK_Icon.png")))
        self.show()

    def mapEvents(self):
        self.upload_raw_data_button.clicked.connect(self.uploadRawData)
        self.raw_data_thread.sendActivity.connect(self.displayActivity)
        self.raw_data_thread.sendMessage.connect(self.displayMessage)

    def displayActivity(self, progress, eta, accepted, rejected, failed, pending):
        self.progress_bar.setValue(progress)
        message = "%d Accepted, %d Rejected, %d Failed, %d Pending. ETA: %s"%(accepted, rejected, failed, pending, eta)
        self.displayMessage(message)
        if progress == 100:
            self.upload_raw_data_button.setEnabled(True)
            self.alertMessage("Completed uploading raw data!", "%s"%message)

    def displayMessage(self, message):
        self.progress_log.append("%s: <b>%s</b>"%(datetime.datetime.now(),message))
        self.progress_log.moveCursor(QtGui.QTextCursor.End)


    def uploadRawData(self):
        self.upload_raw_data_button.setEnabled(False)
        data_file_name = str(QtGui.QFileDialog.getOpenFileName(self,"Open Data File",os.getcwd(),("MS Excel Spreadsheet (*.xlsx)")))
        if data_file_name is not None:
            if os.path.exists(data_file_name):
                xl_file = pd.ExcelFile(data_file_name)
                if "Raw Data" in xl_file.sheet_names:
                    raw_data = xl_file.parse("Raw Data")
                    self.raw_data_thread.setDataFrame(raw_data)
                else:
                    self.alertMessage("Invalid raw data file.","""The given raw data file doesn't seem to have any sheet named "Raw Data".""")
                    
    def alertMessage(self, title, message):
        QtGui.QMessageBox.about(self, title, message)

    def perfromSubmission(self,matched,task):

        njs=0

        ### Progress Bar indicator, deactivate for debug
        if common.debugLevel == 0 :
            term = TerminalController()

        if len(matched)>0:
            common.logger.info(str(len(matched))+" blocks of jobs will be submitted")
            common.logger.debug("Delegating proxy ")
            try:
                common.scheduler.delegateProxy()
            except CrabException:
                common.logger.debug("Proxy delegation failed ")

            for ii in matched:
                common.logger.debug('Submitting jobs '+str(self.sub_jobs[ii]))

                # fix arguments for unique naming of the output
                common._db.updateResubAttribs(self.sub_jobs[ii])

                try:
                    common.scheduler.submit(self.sub_jobs[ii],task)
                except CrabException:
                    common.logger.debug('common.scheduler.submit exception. Job(s) possibly not submitted')
                    raise CrabException("Job not submitted")

                if common.debugLevel == 0 :
                    try: pbar = ProgressBar(term, 'Submitting '+str(len(self.sub_jobs[ii]))+' jobs')
                    except: pbar = None
                if common.debugLevel == 0:
                    if pbar :
                        pbar.update(float(ii+1)/float(len(self.sub_jobs)),'please wait')
                ### check the if the submission succeded Maybe not needed or at least simplified
                sched_Id = common._db.queryRunJob('schedulerId', self.sub_jobs[ii])
                listId=[]
                run_jobToSave = {'status' :'S'}
                listRunField = []
                for j in range(len(self.sub_jobs[ii])):
                    if str(sched_Id[j]) != '':
                        listId.append(self.sub_jobs[ii][j])
                        listRunField.append(run_jobToSave)
                        common.logger.debug("Submitted job # "+ str(self.sub_jobs[ii][j]))
                        njs += 1
                common._db.updateRunJob_(listId, listRunField)
                self.stateChange(listId,"SubSuccess")
                self.SendMLpost(self.sub_jobs[ii])
        else:
            common.logger.info("The whole task doesn't found compatible site ")

        return njs

    def createUI(self, oink_widget_list):
        self.label = QtGui.QLabel("Choose A Widget:")
        self.combo_box_widgets = QtGui.QComboBox()
        self.combo_box_widgets.addItems(oink_widget_list)
        self.button = QtGui.QPushButton("Launch")


        final_layout = QtGui.QHBoxLayout()
        final_layout.addWidget(self.label)
        final_layout.addWidget(self.combo_box_widgets)
        final_layout.addWidget(self.button)
        final_page = QtGui.QWidget()
        final_page.setLayout(final_layout)

        self.progress_bar = ProgressBar()
        self.message = QtGui.QLabel("Loading.....")
        loading_layout = QtGui.QVBoxLayout()
        loading_layout.addWidget(self.progress_bar)
        loading_layout.addWidget(self.message)
        loading_page = QtGui.QWidget()
        loading_page.setLayout(loading_layout)

        self.stacked_widget = QtGui.QStackedWidget()
        self.stacked_widget.addWidget(final_page)
        self.stacked_widget.addWidget(loading_page)

        layout = QtGui.QVBoxLayout()
        layout.addWidget(self.stacked_widget)
        layout.addWidget(Taunter())

        self.setLayout(layout)
        self.setWindowTitle("OINK Widget Chooser")
        icon_file_name_path = os.path.join(MOSES.getPathToImages(),'PORK_Icon.png')
        self.setWindowIcon(QtGui.QIcon(icon_file_name_path))
        self.show()

    def createUI(self):
        self.process_one_button = QtGui.QPushButton("Process Random FSN\nfrom Queue")
    	self.start_progress_button = QtGui.QPushButton("Start Processing All!")
        self.start_progress_button.setFixedSize(200,50)
        buttons_stylesheet = """QPushButton{background-color: #66CD00; color: white}; QPushButton::hover{background-color:#cccce5; color: black}"""
        self.start_progress_button.setStyleSheet(buttons_stylesheet)

    	self.image_viewer_widget = ImageViewerWidget()
    	self.progress_bar = ProgressBar()
    	self.live_progress = QtGui.QWidget()
        self.status_message = QtGui.QLabel("The Cake is a Lie.")
        buttons_layout = QtGui.QHBoxLayout()
        buttons_layout.addWidget(self.process_one_button,0, QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
        buttons_layout.addWidget(self.start_progress_button,0, QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
    	live_progress_layout = QtGui.QVBoxLayout()
    	live_progress_layout.addLayout(buttons_layout, 0)
    	live_progress_layout.addWidget(self.image_viewer_widget,2,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
        live_progress_layout.addSpacing(10)
    	live_progress_layout.addWidget(self.progress_bar,0)
        live_progress_layout.addWidget(self.status_message,0,QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter)
    	self.live_progress.setLayout(live_progress_layout)
    	self.tabs = QtGui.QTabWidget()
    	self.tabs.addTab(self.live_progress,"Live Progress")
    	self.log = QtGui.QTextEdit()
        self.log.setReadOnly(True)
    	self.tabs.addTab(self.log,"Log")
    	layout = QtGui.QHBoxLayout()
    	layout.addWidget(self.tabs)
    	self.setLayout(layout)
    	self.show()

def progressIter( fn, lst ):

    progress = ProgressBar( 0, len(lst), mode='fixed' )
    oldprog = str(progress)

    for n in lst:
        v = fn( n )

        if v:
            progress.update_amount(v)
        else:
            progress.increment_amount()
        if oldprog != str(progress):
            print progress, "\r",
            sys.stdout.flush()
            oldprog=str(progress)
    print '\n'

 def reset(self, title, indeterminate=False):
     if indeterminate != self.indeterminate:
         if self.bar: self.bar.finish()
         self.indeterminate = indeterminate
         if self.bar:
             self.bar = ProgressBar(title=title, indeterminate=indeterminate)
     self.progBar = "[]"   # This holds the progress bar string
     self.width = 40
     self.amount = 0       # When amount == max, we are 100% done

def progressMap( fn, lst ):

    progress = ProgressBar( 0, len(lst), mode='fixed' )
    oldprog = str(progress)

    out = []

    for n in lst:
        v = fn( n )
        out.append( v )

        progress.increment_amount()
        if oldprog != str(progress):
            print progress, "\r",
            sys.stdout.flush()
            oldprog=str(progress)
    print '\n'

    return out