Python CommonUtil.transposed方法代码示例

# 需要导入模块: from RecoLuminosity.LumiDB import CommonUtil [as 别名]
# 或者: from RecoLuminosity.LumiDB.CommonUtil import transposed [as 别名]
def toCSVLumiByLSXing(lumidata,scalefactor,filename):
    '''
    input:{run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),recordedlumi(6),calibratedlumierror(7),{hltpath:[l1name,l1prescale,hltprescale,efflumi]},bxdata,beamdata]}
    output:
    fieldnames=['Run','CMSLS','Delivered(/ub)','Recorded(/ub)','BX']
    '''
    result=[]
    assert(filename)
    fieldnames=['run','ls','delivered(/ub)','recorded(/ub)','bx']
    for run in sorted(lumidata):
        rundata=lumidata[run]
        if rundata is None:
            result.append([run,'n/a','n/a','n/a','n/a'])
            continue
        for lsdata in rundata:
            cmslsnum=lsdata[1]
            if cmslsnum==0:
                continue
            deliveredlumi=lsdata[5]
            recordedlumi=lsdata[6]
            (bxidxlist,bxvaluelist,bxerrorlist)=lsdata[8]
            bxresult=[]
            if bxidxlist and bxvaluelist:
                bxinfo=CommonUtil.transposed([bxidxlist,bxvaluelist])
                bxresult=CommonUtil.flatten([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor,bxinfo])
                result.append(bxresult)
            else:
                result.append([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor])
    r=None
    if filename.upper()=='STDOUT':
        r=sys.stdout
        r.write(','.join(fieldnames)+'\n')
        for l in result:
            r.write(str(l)+'\n')
    else:
        r=csvReporter.csvReporter(filename)
        r.writeRow(fieldnames)
        r.writeRows(result)

# 需要导入模块: from RecoLuminosity.LumiDB import CommonUtil [as 别名]
# 或者: from RecoLuminosity.LumiDB.CommonUtil import transposed [as 别名]
def specificlumiTofile(fillnum,filldata,outdir):
    #
    #input : fillnum
    #        filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]}
    #sorted by bxidx, sorted by lstime inside list
    #check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum
    #
    if not filldata:
        print('empty input data, do nothing for fill ',fillnum)
        return
    timedict={}#{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]}
    filloutdir=os.path.join(outdir,str(fillnum))
    if not os.path.exists(filloutdir):
        os.mkdir(filloutdir)
    for cmsbxidx,perbxdata in filldata.items():
        lhcbucket=0
        if cmsbxidx!=0:
            lhcbucket=(cmsbxidx-1)*10+1
        a=sorted(perbxdata,key=lambda x:x[0])
        filename=str(fillnum)+'_lumi_'+str(lhcbucket)+'_CMS.txt'
        linedata=[]
        for perlsdata in a:
            ts=int(perlsdata[0])
            beamstatusfrac=perlsdata[1]
            lumi=perlsdata[2]
            lumierror=perlsdata[3]
            #beam1intensity=perlsdata[4]
            #beam2intensity=perlsdata[5]
            speclumi=perlsdata[4]
            speclumierror= perlsdata[5]
            if lumi>0:
                linedata.append([ts,beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
            if ts not in timedict:
                timedict[ts]=[]
            timedict[ts].append([beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
        if len(linedata)>10:#at least 10 good ls
            f=open(os.path.join(filloutdir,filename),'w')
            for line in linedata:
                print('%d\t%e\t%e\t%e\t%e\t%e'%(line[0],line[1],line[2],line[3],line[4],line[5]), file=f)
            f.close()
    #print 'writing avg file'
    summaryfilename=str(fillnum)+'_lumi_CMS.txt'
    f=None
    lstimes=sorted(timedict.keys())
    fillseg=[]
    lscounter=0
    for lstime in lstimes:
        allvalues=timedict[lstime]
        transposedvalues=CommonUtil.transposed(allvalues,0.0)
        bstatfrac=transposedvalues[0][0]#beamstatus does not change with bx position
        lumivals=transposedvalues[1]
        lumitot=sum(lumivals)
        if bstatfrac==1.0 :
            fillseg.append([lstime,lumitot])
        lumierrs=transposedvalues[2]
        lumierrortot=math.sqrt(sum(map(lambda x:x**2,lumierrs)))
        specificvals=transposedvalues[3]
        specificavg=sum(specificvals)/float(len(specificvals))#avg spec lumi
        specificerrs=transposedvalues[4]
        specifictoterr=math.sqrt(sum(map(lambda x:x**2,specificerrs)))
        specificerravg=specifictoterr/float(len(specificvals))
        if lscounter==0:
            f=open(os.path.join(filloutdir,summaryfilename),'w')
        lscounter+=1
        print('%d\t%e\t%e\t%e\t%e\t%e'%(lstime,bstatfrac,lumitot,lumierrortot,specificavg,specificerravg), file=f)
    if f is not None:
        f.close()
    #print 'writing summary file'
    fillsummaryfilename=str(fillnum)+'_bxsum_CMS.txt'
    f=open(os.path.join(filloutdir,fillsummaryfilename),'w')    
    if len(fillseg)==0:
        print('%s'%('#no stable beams'), file=f)
        f.close()
        return
    previoustime=fillseg[0][0]
    boundarytime=fillseg[0][0]
    #print 'boundary time ',boundarytime
    summaryls={}
    summaryls[boundarytime]=[]
    for [lstime,lumitot] in fillseg:#fillseg is everything with stable beam flag
        if lstime-previoustime>50.0:
            boundarytime=lstime
            #print 'found new boundary ',boundarytime
            summaryls[boundarytime]=[]
     #   print 'appending ',boundarytime,lstime,lumitot
        summaryls[boundarytime].append([lstime,lumitot])
        previoustime=lstime
    #print summaryls
   
    summarylstimes=summaryls.keys()
    summarylstimes.sort()
    lumip=lumiParameters.ParametersObject()
    for bts in summarylstimes:
        startts=bts
        tsdatainseg=summaryls[bts]
        #print 'tsdatainseg ',tsdatainseg
        stopts=tsdatainseg[-1][0]
        plu=max(CommonUtil.transposed(tsdatainseg,0.0)[1])
        lui=sum(CommonUtil.transposed(tsdatainseg,0.0)[1])*lumip.lslengthsec()
        print('%d\t%d\t%e\t%e'%(startts,stopts,plu,lui), file=f)
#.........这里部分代码省略.........

# 需要导入模块: from RecoLuminosity.LumiDB import CommonUtil [as 别名]
# 或者: from RecoLuminosity.LumiDB.CommonUtil import transposed [as 别名]

#.........这里部分代码省略.........
                if args.outputfile :
                    reporter.writeRow([fill,run,lumiDict[run][0],lumiDict[run][1]])   
        #print 'input fillDict ',len(fillDict.keys()),fillDict
        m.plotSumX_Fill(xdata,ydata,fillDict,yscale='linear')
        mlog.plotSumX_Fill(xdata,ydata,fillDict,yscale='log')
    elif args.action == 'time' : 
        lumiDict={}
        lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
        #lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus='STABLE BEAMS')
        xdata={}#{run:[starttime,stoptime]}
        ydata={}
        ydata['Delivered']=[]
        ydata['Recorded']=[]
        keylist=lumiDict.keys()
        keylist.sort()
        if args.outputfile:
            reporter=csvReporter.csvReporter(ofilename)
            fieldnames=['run','starttime','stoptime','delivered','recorded']
            reporter.writeRow(fieldnames)
        for run in keylist:
            ydata['Delivered'].append(lumiDict[run][0])
            ydata['Recorded'].append(lumiDict[run][1])
            starttime=runDict[run][0]
            stoptime=runDict[run][1]
            xdata[run]=[starttime,stoptime]
            if args.outputfile :
                reporter.writeRow([run,starttime,stoptime,lumiDict[run][0],lumiDict[run][1]])
        m.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='linear')
        mlog.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='log')
    elif args.action == 'perday':
        daydict={}#{day:[[run,cmslsnum,lsstarttime,delivered,recorded]]}
        lumibyls=getLumiOrderByLS(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
        #lumibyls [[runnumber,runstarttime,lsnum,lsstarttime,delivered,recorded,recordedinpath]]
        if args.outputfile:
            reporter=csvReporter.csvReporter(ofilename)
            fieldnames=['day','begrunls','endrunls','delivered','recorded']
            reporter.writeRow(fieldnames)
        beginfo=[lumibyls[0][3],str(lumibyls[0][0])+':'+str(lumibyls[0][2])]
        endinfo=[lumibyls[-1][3],str(lumibyls[-1][0])+':'+str(lumibyls[-1][2])]
        for perlsdata in lumibyls:
            lsstarttime=perlsdata[3]
            delivered=perlsdata[4]
            recorded=perlsdata[5]
            day=lsstarttime.toordinal()
            if not daydict.has_key(day):
                daydict[day]=[]
            daydict[day].append([delivered,recorded])
        days=daydict.keys()
        days.sort()
        daymin=days[0]
        daymax=days[-1]
        #alldays=range(daymin,daymax+1)
        resultbyday={}
        resultbyday['Delivered']=[]
        resultbyday['Recorded']=[]
        #for day in days:
        #print 'day min ',daymin
        #print 'day max ',daymax
        for day in range(daymin,daymax+1):
            if not daydict.has_key(day):
                delivered=0.0
                recorded=0.0
            else:
                daydata=daydict[day]
                mytransposed=CommonUtil.transposed(daydata,defaultval=0.0)
                delivered=sum(mytransposed[0])
                recorded=sum(mytransposed[1])
            resultbyday['Delivered'].append(delivered)
            resultbyday['Recorded'].append(recorded)
            if args.outputfile:
                reporter.writeRow([day,beginfo[1],endinfo[1],delivered,recorded])
        #print 'beginfo ',beginfo
        #print 'endinfo ',endinfo
        #print resultbyday
        m.plotPerdayX_Time( range(daymin,daymax+1) ,resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='linear')
        mlog.plotPerdayX_Time( range(daymin,daymax+1),resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='log')
    else:
        raise Exception,'must specify the type of x-axi'

    del session
    del svc

    if args.batch and args.yscale=='linear':
        m.drawPNG(args.batch)
    elif args.batch and args.yscale=='log':
        mlog.drawPNG(args.batch)
    elif args.batch and args.yscale=='both':
        m.drawPNG(args.batch)
        basename,extension=os.path.splitext(args.batch)
        logfilename=basename+'_log'+extension        
        mlog.drawPNG(logfilename)
    else:
        if not args.interactive:
            return
        if args.interactive is True and args.yscale=='linear':
            m.drawInteractive()
        elif args.interactive is True and args.yscale=='log':
            mlog.drawInteractive()
        else:
            raise Exception('unsupported yscale for interactive mode : '+args.yscale)

# 需要导入模块: from RecoLuminosity.LumiDB import CommonUtil [as 别名]
# 或者: from RecoLuminosity.LumiDB.CommonUtil import transposed [as 别名]
def getLumiOrderByLS(dbsession,c,runList,selectionDict,hltpath='',beamstatus=None,beamenergy=None,beamfluctuation=None,finecorrections=None):
    '''
    input:  runList[runnum], selectionDict{runnum:[ls]}
    output: [[runnumber,runstarttime,lsnum,lsstarttime,delivered,recorded,recordedinpath]]
    '''
    #print 'getLumiOrderByLS selectionDict seen ',selectionDict
    t=lumiTime.lumiTime()
    result=[]#[[runnumber,runstarttime,lsnum,lsstarttime,delivered,recorded]]
    dbsession.transaction().start(True)
    sortedresult=[]
    #print 'runlist ',runList
    for runnum in runList:
        delivered=0.0
        recorded=0.0       
        #print 'looking for run ',runnum
        q=dbsession.nominalSchema().newQuery()
        runsummary=lumiQueryAPI.runsummaryByrun(q,runnum)
        del q
        runstarttimeStr=runsummary[3]
        if len(runstarttimeStr)==0:
            if c.VERBOSE: print 'warning request run ',runnum,' has no runsummary, skip'
            continue
        if len(selectionDict)!=0 and not selectionDict.has_key(runnum):
            if runnum<max(selectionDict.keys()):
                result.append([runnum,runstarttimeStr,1,t.StrToDatetime(runstarttimeStr),0.0,0.0])
            continue
        #print 'runsummary ',runsummary
        lumitrginfo={}
        q=dbsession.nominalSchema().newQuery()
        if finecorrections and finecorrections[runnum]:
            lumitrginfo=lumiQueryAPI.lumisummarytrgbitzeroByrun(q,runnum,c.LUMIVERSION,beamstatus,beamenergy,beamfluctuation,finecorrections=finecorrections[runnum]) #q2
        else:
            lumitrginfo=lumiQueryAPI.lumisummarytrgbitzeroByrun(q,runnum,c.LUMIVERSION,beamstatus,beamenergy,beamfluctuation) #q2
        del q
        #print 'lumitrginfo ',lumitrginfo
        if len(lumitrginfo)==0: #if no qualified cross lumi-trg found, try lumionly
            #result.append([runnum,runstarttimeStr,1,t.StrToDatetime(runstarttimeStr),0.0,0.0])
            q=dbsession.nominalSchema().newQuery()
            if finecorrections and finecorrections[runnum]:
                lumiinfobyrun=lumiQueryAPI.lumisummaryByrun(q,runnum,c.LUMIVERSION,beamstatus,beamenergy,beamfluctuation,finecorrections=finecorrections[runnum]) #q3
            else:
                lumiinfobyrun=lumiQueryAPI.lumisummaryByrun(q,runnum,c.LUMIVERSION,beamstatus,beamenergy,beamfluctuation)
            del q
            if len(lumiinfobyrun)!=0: #if lumionly has qualified data means trg has no data
                print 'warning request run ',runnum,' has no trigger data, calculate delivered only'
                for perlsdata in lumiinfobyrun:
                    cmslsnum=perlsdata[0]
                    instlumi=perlsdata[1]
                    norbit=perlsdata[2]
                    startorbit=perlsdata[3]
                    lsstarttime=t.OrbitToTime(runstarttimeStr,startorbit)
                    lslength=t.bunchspace_s*t.nbx*norbit
                    delivered=instlumi*lslength
                    result.append([runnum,runstarttimeStr,cmslsnum,lsstarttime,delivered,0.0])
            else:
                #print 'run '+str(runnum)+' has no qualified data '
                lsstarttime=t.OrbitToTime(runstarttimeStr,0)
                result.append([runnum,runstarttimeStr,1,lsstarttime,0.0,0.0])
        else:
            norbits=lumitrginfo.values()[0][1]
            lslength=t.bunchspace_s*t.nbx*norbits
            trgbitinfo={}
            for cmslsnum,valuelist in lumitrginfo.items():
                instlumi=valuelist[0]
                startorbit=valuelist[2]
                bitzero=valuelist[5]
                deadcount=valuelist[6]
                prescale=valuelist[-1]
                lsstarttime=t.OrbitToTime(runstarttimeStr,startorbit)        
                if len(selectionDict)!=0 and not (cmslsnum in selectionDict[runnum]):
                   #if there's a selection list but cmslsnum is not selected,set to 0
                   result.append([runnum,runstarttimeStr,cmslsnum,lsstarttime,0.0,0.0])
                   continue
                delivered=instlumi*lslength
                if valuelist[5]==0:#bitzero==0 means no beam,do nothing
                    recorded=0.0
                else:
                    deadfrac=float(deadcount)/float(float(bitzero)*float(prescale))
                    if(deadfrac<1.0):
                        recorded=delivered*(1.0-deadfrac)
                result.append([runnum,runstarttimeStr,cmslsnum,lsstarttime,delivered,recorded])
                #print 'result : ',result
    dbsession.transaction().commit()
    transposedResult=CommonUtil.transposed(result)
    lstimes=transposedResult[3]
    lstimes.sort()
    for idx,lstime in enumerate(lstimes):
        sortedresult.append(result[idx])
    if c.VERBOSE:
        print sortedresult
    return sortedresult           

# 需要导入模块: from RecoLuminosity.LumiDB import CommonUtil [as 别名]
# 或者: from RecoLuminosity.LumiDB.CommonUtil import transposed [as 别名]

#.........这里部分代码省略.........
    #print 'runList ',runList
    #print 'runDict ', runDict
    finecorrections=None
    if args.withFineCorrection:
        schema=session.nominalSchema()
        session.transaction().start(True)
        finecorrections=lumiCorrections.correctionsForRange(schema,runList)
        session.transaction().commit()      
    fig=Figure(figsize=(6,4.5),dpi=100)
    m=matplotRender.matplotRender(fig)

    logfig=Figure(figsize=(6,4.5),dpi=100)
    mlog=matplotRender.matplotRender(logfig)
    
    if args.action == 'peakperday':
        l=lumiTime.lumiTime()
        lumiperls=getInstLumiPerLS(session,c,runList,selectionDict,finecorrections=finecorrections)
        if args.outputfile:
            reporter=csvReporter.csvReporter(ofilename)
            fieldnames=['day','run','lsnum','maxinstlumi']
            reporter.writeRow(fieldnames)
        #minDay=minTime.toordinal()
        #maxDay=maxTime.toordinal()
        daydict={}#{day:[[run,lsnum,instlumi]]}
        result={}#{day:[maxrun,maxlsnum,maxinstlumi]}
        for lsdata in lumiperls:
            runnumber=lsdata[0]
            lsnum=lsdata[1]
            runstarttimeStr=lsdata[-2]#note: it is a string!!
            startorbit=lsdata[5]
            deliveredInst=lsdata[2]
            lsstarttime=l.OrbitToTime(runstarttimeStr,startorbit)
            day=lsstarttime.toordinal()
            if not daydict.has_key(day):
                daydict[day]=[]
            daydict[day].append([runnumber,lsnum,deliveredInst])
        days=daydict.keys()
        days.sort()
        for day in days:
            daydata=daydict[day]
            transposeddata=CommonUtil.transposed(daydata,defaultval=0.0)
            todaysmaxinst=max(transposeddata[2])
            todaysmaxidx=transposeddata[2].index(todaysmaxinst)
            todaysmaxrun=transposeddata[0][todaysmaxidx]
            todaysmaxls=transposeddata[1][todaysmaxidx]
            result[day]=[todaysmaxrun,todaysmaxls,todaysmaxinst]
            if args.outputfile :
                reporter.writeRow([day,todaysmaxrun,todaysmaxls,todaysmaxinst])
        m.plotPeakPerday_Time(result,startRunTime,stopRunTime,annotateBoundaryRunnum=args.annotateboundary,yscale='linear')
        mlog.plotPeakPerday_Time(result,startRunTime,stopRunTime,annotateBoundaryRunnum=args.annotateboundary,yscale='log')
        
    if args.action == 'run':
        runnumber=runList[0]
        if finecorrections and finecorrections[runnumber]:
            lumiperrun=getLumiPerRun(session,c,runnumber,finecorrections=finecorrections[runnumber])#[[lsnumber,deliveredInst,recordedInst,norbit,startorbit,fillnum,runstarttime,runstoptime]]
        else:
            lumiperrun=getLumiPerRun(session,c,runnumber)
        #print 'lumiperrun ',lumiperrun
        xdata=[]#[runnumber,fillnum,norbit,stattime,stoptime,totalls,ncmsls]
        ydata={}#{label:[instlumi]}
        ydata['Delivered']=[]
        ydata['Recorded']=[]
        norbit=lumiperrun[0][3]
        fillnum=lumiperrun[0][-3]
        starttime=lumiperrun[0][-2]
        stoptime=lumiperrun[0][-1]
        ncmsls=0
        totalls=len(lumiperrun)
        for lsdata in lumiperrun:
            lsnumber=lsdata[0]
            if lsnumber!=0:
                ncmsls+=1
            deliveredInst=lsdata[1]
            recordedInst=lsdata[2]
            ydata['Delivered'].append(deliveredInst)
            ydata['Recorded'].append(recordedInst)
        xdata=[runnumber,fillnum,norbit,starttime,stoptime,totalls,ncmsls]
        m.plotInst_RunLS(xdata,ydata)
    del session
    del svc
    if args.batch and args.yscale=='linear':
        m.drawPNG(args.batch)
    elif  args.batch and args.yscale=='log':
        mlog.drawPNG(args.batch)
    elif args.batch and args.yscale=='both':
        m.drawPNG(args.batch)
        basename,extension=os.path.splitext(args.batch)
        logfilename=basename+'_log'+extension        
        mlog.drawPNG(logfilename)
    else:
        raise Exception('unsupported yscale for batch mode : '+args.yscale)
    
    if not args.interactive:
        return
    if args.interactive is True and args.yscale=='linear':
        m.drawInteractive()
    elif args.interactive is True and args.yscale=='log':
        mlog.drawInteractive()
    else:
        raise Exception('unsupported yscale for interactive mode : '+args.yscale)