Python tables.taql函数代码示例

    def getMessages(self, start_time, end_time, start_freq, end_freq, iteration="last"):
        messagesDict={}       # create an empty dictionary
        # return all iterations (default behaviour)
        if iteration == "all":
           messagesDict["result"]="all"

           # Loop over all iterations
           for iter in range(1, self.getMaxIter()+1):
                 taqlcmd="SELECT * FROM " + self.tablename + " WHERE STARTTIME>=" + str(start_time) + " AND ENDTIME<=" + str(end_time) + " AND STARTFREQ>=" + str(start_freq) + " AND ENDFREQ<=" + str(end_freq) + " AND ITER=" + str(iter)
                 result=pt.taql(taqlcmd)           # execute TaQL command
                 messagesDict[iter]=result.getcol("MESSAGE")

        # return the last iteration only
        elif iteration == "Last" or iteration == "last":
           #print "readCells(): last"        # DEBUG
           messagesDict["result"]="last"

           taqlcmd="SELECT * FROM " + self.tablename + " WHERE STARTTIME>=" + str(start_time) + " AND ENDTIME<=" + str(end_time) + " AND STARTFREQ>=" + str(start_freq) + " AND ENDFREQ<=" + str(end_freq) + " AND LASTITER=TRUE"
           result=pt.taql(taqlcmd)           # execute TaQL command
           
           print "result.nrows() = ", result.nrows()
           
           messagesDict["last"]=result.getcol("MESSAGE")

        # return only a particular iteration
        elif type(iteration).__name__ == "int":
            #print "iteration: ", iteration    # DEBUG
            messagesDict["result"]="iteration"
            taqlcmd="SELECT * FROM " + self.tablename + + " WHERE STARTTIME>=" + str(start_time) + " AND ENDTIME<=" + str(end_time) + " AND STARTFREQ=" + str(start_freq) + " AND ENDFREQ=" + str(end_freq) + " AND ITER=" + str(iteration) + " ORDERBY STARTFREQ"
            result=pt.taql(taqlcmd)        # execute TaQL command      
            
            messagesDict[iteration]=result.getcol("MESSAGE")

        return messagesDict

def addcol(ms, incol, outcol):
    if outcol not in ms.colnames():
        logging.info('Adding column: '+outcol)
        coldmi = ms.getdminfo(incol)
        coldmi['NAME'] = outcol
        ms.addcols(pt.makecoldesc(outcol, ms.getcoldesc(incol)), coldmi)
    if outcol != incol:
        # copy columns val
        logging.info('Set '+outcol+'='+incol)
        pt.taql("update $ms set "+outcol+"="+incol)

    def readTimeColumn(self, parameter, iteration="all"):
        print "readTimeColumn(self, parameter, iteration=", iteration ,"):"   # DEBUG
        
        # Get first all unique time slots
        if self.timeSlots.nrows()==0:
            self.timeSlots=self.getTimeSlots()

        # Get MAXITER first
        maxIter=self.getMaxIter()
        print "maxIter: ", maxIter

        parmsDict={}

        # return all iterations (default behaviour)
        if iteration == "all":
           parmsDict["result"]="all"
   
           # Loop over all iterations
           for iter in range(1, maxIter+1):
                 taqlcmd="SELECT DISTINCT STARTTIME, ENDTIME, ITER, " + parameter + " FROM " + self.tablename + " WHERE ITER=" + str(iter)
                 selection=pt.taql(taqlcmd)              # execute TaQL command
                 parmIter=selection.getcol(parameter)    # select column with wanted parameter
                 print "readTimeColumn-type(parmIter): ", type(parmIter)
                 parmsDict[iter]=parmIter
           return parmsDict

        # return the last iteration only
        elif iteration == "Last" or iteration == "last":
           parmsDict["result"]="last"

           taqlcmd="SELECT DISTINCT STARTTIME, ENDTIME, ITER, " + parameter + " FROM " + self.tablename + " WHERE LASTITER=TRUE"
           selection=pt.taql(taqlcmd)           # execute TaQL command
           parmDict["last"]=selection.getcol(parameter)    # select column with wanted parameter

           return parmsDict

        # return only a particular iteration
        elif type(iteration).__name__==int:
            parmsDict["result"]="iteration"

            taqlcmd="SELECT DISTINCT STARTTIME, ENDTIME, ITER FROM " + self.tablename + " WHERE ITER=" + str(iteration) + " ORDERBY STARTTIME"
            selection=pt.taql(taqlcmd)           # execute TaQL command      
            parmsDict[iteration]=selection.getcol(parameter)    # select column with wanted parameter

            return parmsDict

        else:
            parmsDict["result"]=False
            return parmsDict

    def getCorrMatrix(self, start_time, end_time, start_freq, end_freq, getStartTimes=True, getRank=False):
        start_time, end_time=self.fuzzyTime(start_time, end_time)
        start_freq, end_freq=self.fuzzyFreq(start_freq, end_freq)

        corrMatrix=[]    # list to contain returned corrMatrices

        # LASTITER=TRUE (Correlation matrix is only recorded by the solver for the last iteration)
        taqlcmd="SELECT STARTTIME, CORRMATRIX FROM " + self.tablename + " WHERE STARTTIME >= "+ str(start_time) + " AND ENDTIME <= " + str(end_time) + " AND STARTFREQ >= " + str(start_freq) + " AND ENDFREQ <= " + str(end_freq) + " AND LASTITER=TRUE"

        result=pt.taql(taqlcmd)
        rank=self.getRank(start_time, end_time, start_freq, end_freq)          # get the RANK from this cell
        #rankDef=self.getRankDef(start_time, end_time, start_freq, end_freq)    # rank deficiency

        #print "solverQuery::getCorrMatrix() result.nrows() = ", result.nrows()   # DEBUG

        # This is needed if we get back more than one result (but is buggy now)
        #if self.type=="PERITERATION_CORRMATRIX":
        #    result=result[1]['CORRMATRIX']
        #else:
        #    result=result[0]['CORRMATRIX']
  
        # The Corrmatrix will only be for (N+1)th iteration
        if result.nrows()==1:
          corrMatrix=result[0]['CORRMATRIX']  # select CORRMATRIX and write to numpy 1D-array
        else:
          corrMatrix=result[1]['CORRMATRIX']  # select CORRMATRIX and write to numpy 1D-array

        if getStartTimes==True and getRank==True:
            starttimes=result[1].getcol('STARTTIME')  # also select starttimes
            return corrMatrix, starttimes, getRank
        elif getStartTimes==False and getRank==True: 
            return corrMatrix, getRank
        else:
            return corrMatrix

 def setFreqs(self):
     if len(self.frequencies) == 0:
         taqlcmd="SELECT UNIQUE STARTFREQ, ENDFREQ FROM " + self.tablename
         self.frequencies=pt.taql(taqlcmd)
         
         self.startFreqs=self.frequencies.getcol("STARTFREQ")
         self.endFreqs=self.frequencies.getcol("ENDFREQ")

    def readCell(self, start_time, end_time, start_freq, end_freq, iteration="Last"):
        print "readCell(self, start_time, end_time, start_freq, end_freq, iteration=Last)"  # DEBUG

        start_time, end_time=self.fuzzyTime(start_time, end_time)
        start_freq, end_freq=self.fuzzyFreq(start_freq, end_freq)

        cellDict={}

        # return all iterations (default behaviour)
        if iteration == "all":
           cellDict["result"]="all"                   # give type of result

           # Loop over all iterations
           for iter in range(1, self.getMaxIter()):
                 taqlcmd="SELECT * FROM " + self.tablename + " WHERE STARTFREQ >=" + str(start_freq) + " AND ENDFREQ <= " + str(end_freq) + " AND ITER = " + str(iter)
                 cell[iter]=pt.taql(taqlcmd)           # execute TaQL command
           return cell

        # return the last iteration only
        elif iteration == "Last" or iteration == "last":
           cellDict["result"]="last"                   # give type of result

           # Loop over all iterations
           taqlcmd="SELECT * FROM " + self.tablename + " WHERE LASTITER=", str(iter)
           selection=pt.taql(taqlcmd)           # execute TaQL command
           cellDict["last"]=selection

           return cellDict

        # return only a particular iteration
        elif isinstance(iteration, int):
            cellDict["result"]="iteration"                   # give type of result

            taqlcmd="SELECT * FROM " + self.tablename + " WHERE STARTFREQ=" + str(start_freq) + " AND ENDFREQ=" + str(end_freq) + " AND ITER=" + str(iteration) + " ORDERBY STARTFREQ"
            selection=pt.taql(taqlcmd)           # execute TaQL command      

            cellDict[iteration]=selection

            return cellDict

        else:
            print "readCell(): unknown iteration"
            cellDict["result"]="False"


            return cellDict

    def readFreqColumn(self, parameter, iteration="all"):
        #print "readFreqColumn(self, parameter, iteration=", iteration, ":"    # DEBUG

        # Get first all unique frequencies
        if len(self.frequencies)==0:
            self.frequencies=getFreqs()

        # Get MAXITER first
        #maxIter=pt.tablecolumn(self.solverTable, "MAXITER")[0]

        parmsDict={}       # create an empty dictionary

        # return all iterations (default behaviour)
        if iteration == "all":
           parmsDict["result"]="all"

           # Loop over all iterations
           for iter in range(1, self.getMaxIter()+1):
                 taqlcmd="SELECT DISTINCT STARTFREQ, ENDFREQ, ITER, " + parameter + " FROM " + self.tablename + " WHERE ITER=" + str(iter)
                 selection=pt.taql(taqlcmd)              # execute TaQL command
                 parmIter=selection.getcol(parameter)    # select column with wanted parameter
                 parmsDict[iter]=parmIter                    # write into dictionary__
           return parmsDict

        # return the last iteration only
        elif iteration == "Last" or iteration == "last":
            parmsDict["result"]="last"

            taqlcmd="SELECT DISTINCT STARTFREQ, ENDFREQ, " + parameter + " FROM " + self.tablename + " WHERE LASTITER=TRUE"
            selection=pt.taql(taqlcmd)           # execute TaQL command
            parmsDict["last"]=selection.getcol(parameter)    # select column with wanted parameter
            return parmsDict

        # return only a particular iteration
        elif type(iteration).__name__ == "int":
            parmsDict["result"]="iteration"

            taqlcmd="SELECT " + parameter + " FROM " + self.tablename + " WHERE ITER=" + str(iteration) + " ORDERBY STARTFREQ"
            #print "taqlcmd: ", taqlcmd           # DEBUG
            selection=pt.taql(taqlcmd)           # execute TaQL command      
            parmsDict[iteration]=selection.getcol(parameter)    # select column with wanted parameter

            return parmsDict
        else:
            parmsDict["result"]=False
            return parmsDict

def addSourceTable (image, sourcedbName, minTime, maxTime):
    # Create the table using TaQL.
    tab = pt.taql ("create table '" + image.name() + "/LOFAR_SOURCE' " + 
                   "SOURCE_ID int, \TIME double, INTERVAL double, " +
                   "NUM_LINES int, NAME string, " +
                   "DIRECTION double shape=[2], " +
                   "PROPER_MOTION double shape=[2], " +
                   "FLUX double shape=[4], " +
                   "SPINX double, REF_FREQUENCY double, " +
                   "SHAPE double shape=[3]")
    tab.putcolkeyword ("TIME", "QuantumUnits", ["s"])
    tab.putcolkeyword ("INTERVAL", "QuantumUnits", ["s"])
    tab.putcolkeyword ("DIRECTION", "QuantumUnits", ["rad"])
    tab.putcolkeyword ("PROPER_MOTION", "QuantumUnits", ["rad/s"])
    tab.putcolkeyword ("FLUX", "QuantumUnits", ["Jy"])
    tab.putcolkeyword ("REF_FREQUENCY", "QuantumUnits", ["MHz"])
    tab.putcolkeyword ("SHAPE", "QuantumUnits", ["rad", "rad", "rad"])
    tab.putcolkeyword ("TIME", "MEASINFO", {"Ref":"UTC", "type":"epoch"})
    tab.putcolkeyword ("DIRECTION", "MEASINFO", {"Ref":"J2000", "type":"direction"})
    tab.flush()
    image.putkeyword ("ATTRGROUPS." + "LOFAR_SOURCE", tab)
    # Get all parameters from the source parmdb.
    midtime = (minTime + maxTime) / 2
    inttime = maxTime - minTime
    sourcedb = pdb.parmdb(sourcedbName)
    # Get all source names by getting the Ra parms from DEFAULTVALUES
    names = [name[3:] for name in sourcedb.getDefNames ("Ra:*")]
    values = sourcedb.getDefValues()
    sourcedb = 0   # close
    row = 0
    tab.addrows (len(names))
    # Add the info of all sources.
    # The field names below are as used in SourceDB.
    fldnames = ["Ra", "Dec", "I", "Q", "U", "V", "SpectralIndex:0",
                "ReferenceFrequency", "Orientation", "MajorAxis", "MinorAxis"]
    vals = [0. for fld in fldnames]
    for name in names:
        for i in range(len(fldnames)):
            key = fldnames[i] + ":" + name
            if values.has_key (key):
                vals[i] = values[key][0][0]
            else:
                vals[i] = 0.
        tab.putcell ("SOURCE_ID", row, row)
        tab.putcell ("TIME", row, midtime);
        tab.putcell ("INTERVAL", row, inttime);
        tab.putcell ("NUM_LINES", row, 0);
        tab.putcell ("NAME", row, name);
        tab.putcell ("DIRECTION", row, vals[:2]);
        tab.putcell ("PROPER_MOTION", row, (0.,0.));
        tab.putcell ("FLUX", row, vals[2:6]);
        tab.putcell ("SPINX", row, vals[6]);
        tab.putcell ("REF_FREQUENCY", row, vals[7]);
        tab.putcell ("SHAPE", row, vals[8:11]);
        row += 1
    # Ready.
    tab.close()
    print "Added subtable LOFAR_SOURCE containing", row, "rows"

    def run(self, infile):
        with log_time(self.logger):
            if os.path.exists(infile):
                self.logger.info("Processing %s" % (infile))
            else:
                self.logger.error("Dataset %s does not exist" % (infile))
                return 1

            try:
                self.outputs['start_time'] = taql(
                    "CALC MIN([SELECT TIME from %s])" % infile
                )[0]
                self.outputs['end_time'] = taql(
                    "CALC MAX([SELECT TIME from %s])" % infile
                )[0]
            except Exception, e:
                self.logger.error(str(e))
                return 1

def read_ms(logger, msname, ateam, diameter=None):
    def get_station_diameter(table):
        histable = pt.table(table.getkeyword('HISTORY'), ack=False)
        for line in histable.getcell('APP_PARAMS', 0):
            try:
                key, value = line.split("=")
            except:
                pass
            if key == "Observation.antennaSet":
                antenna_set = value
                break
        if antenna_set == "LBA_INNER":
            logger.debug("LBA_INNER mode")
            return STATION_DIAMETER["LBA_INNER"]
        elif antenna_set[:3] == "LBA":
            logger.debug("LBA_(OUTER,SPARSE,X,Y) mode")
            return STATION_DIAMETER["LBA"]
        elif antenna_set[:3] == "HBA":
            logger.debug("HBA mode")
            return STATION_DIAMETER["HBA"]
        else:
            logger.error("Failed to identify antenna set")

    def field_size_ateam(table):
        logging.debug('Computing field size for A-team')
        fieldtable = table.getkeyword('FIELD').split()[1]
        taqloutput = pt.taql("calc from %s calc max(angdist (DELAY_DIR[0,], [%s]))" % (fieldtable, ", ".join(",".join(src) for src in ATEAM))  )
        return taqloutput[0]

    def field_size_nominal(table, wavelength, diameter):
        if not diameter:
            diameter = get_station_diameter(table)
        logger.debug("Station diameter %f m" % diameter)
        return 1.22*wavelength/diameter

    t = pt.table(msname, readonly=True, ack=False)
    interval = t.getcell('INTERVAL', 0)
    swtable = t.getkeyword('SPECTRAL_WINDOW')
    tsw = pt.table(swtable, readonly=True, ack=False)
    freq = tsw.getcell('REF_FREQUENCY', 0)
    wavelength = 299792458./freq
    maxbl = pt.taql("calc sqrt(max([select sumsqr(UVW[0:1]) from %s]))" % msname)[0] / wavelength
    chwidth = tsw.getcell('CHAN_WIDTH', 0)[0]

    if ateam:
        fieldsize = field_size_ateam(t)
    else:
        fieldsize = field_size_nominal(t, wavelength, diameter)

    logger.debug('Frequency is %f MHz'%(freq/1.e6))
    logger.debug('Wavelength is %f m'%(wavelength))
    logger.debug('Maximum baseline length is %f m = %f lambdas'%(maxbl*wavelength,maxbl))
    logger.debug('Integration time is %f sec'%(interval))
    logger.debug('Channel width is %f Hz'%(chwidth))
    logger.debug('Field size is %f degrees'%(fieldsize*180./3.14159))

    return fieldsize, maxbl, freq, interval, chwidth

    def getRankDef(self, start_time=None, end_time=None, start_freq=None, end_freq=None):
        if start_time == None or end_time == None or start_freq == None or end_freq == None:
            rank=self.readParameterIdx("RANK", 0)
        else:
            taqlcmd="SELECT RANKDEF FROM " + self.tablename + " WHERE STARTTIME >= " + str(start_time) + " AND ENDTIME <= " + str(end_time) + " AND STARTFREQ >= " + str(start_freq) + " AND ENDFREQ <= " + str(end_freq) + " AND LASTITER=TRUE"
            result=pt.taql(taqlcmd)
            rankdef=result.getcol("RANKDEF")

        return rankdef

    def compareColumn(self, columnname, taql=False):
        if self.verbose:
            print "Comparing "+  bcolors.OKBLUE + columnname + bcolors.ENDC + " columns." # DEBUG

        passed=False
        errorcount=0                                # counter that counts rows with differying columns

        if taql==False:                             # If taql is not to be used for comparison, use numpy difference
          if self.debug:
            print "compareColumn() using numpy" 

          reftab=pt.table(self.MS)                # Open reference MS in readonly mode
          testtab=pt.table(self.test_MS)          # Open test MS in readonly mode     
     
          tc_ref=reftab.col(columnname)           # get column in reference table as numpy array
          tc_test=testtab.col(columnname)         # get column in test table as numpy array
  
          nrows=testtab.nrows()                  
          for i in progressbar( range(0, nrows-1), "comparing " + columnname + " ", 60):
              difference = numpy.max(abs(tc_test[i] - tc_ref[i]))    # Use numpy's ability to substract arrays from each other
              #sum=numpy.sum(difference)
              
              #if sum > (self.acceptancelimit/len(difference)):     # determine if this failed the test
              if difference > self.acceptancelimit:                 # determine if this failed the test
                  passed=False
              else:
                  passed=True
  
          reftab.close()
          testtab.close()
        else:
            if self.debug:
              print "compareColumn() using TaQL"          # DEBUG
  
            self.addRefColumnToTesttab(columnname)      # add reference table column as forward column
        
            testcolumnname = "test_" + columnname       # create name which is used in test_MS if refcolum was added
        
            # Loop over columns, compute and check difference (must be within self.acceptancelimit)            
            # use TaQL for this? How to select from two tables? TODO: check this!
            
#            taqlcmd = "SELECT * FROM '" + self.test_MS + "' WHERE !all(NEAR(Real("+columnname+"), Real("+testcolumnname+")) AND NEAR(Imag("+columnname+"), Imag("+testcolumnname+")))"
#            errorcount = result.nrows()
            taqlcmd = "SELECT * FROM '" + self.test_MS + "' WHERE !all(NEARABS(Real("+columnname+"), Real("+testcolumnname+")," + str(self.acceptancelimit) + ") AND NEARABS(Imag("+columnname+"), Imag("+testcolumnname+"),"+ str(self.acceptancelimit) +"))"
#            print "taqlcmd = ", taqlcmd     # DEBUG
            errorcount=pt.taql(taqlcmd).nrows()            
            
            if self.verbose or self.debug:
              print "errorcount = ", errorcount         # display number of errors=No. of rows

            # If test_MS COLUMN and reference COLUMN have any discrepancy...            
            if errorcount > 0:
                passed=False      # ... the test is failed
            else:
                passed=True
        return passed

    def _convert_polarization(self, time_slice_filtered_path_list):
        """
        # convert to circular polarization 
        # method based on input from Javier Moldon <[email protected]>
        """
        for time_slice in time_slice_filtered_path_list:
            #apply the polarization to each ms
            try:
                opened_ms=pt.taql(
                    "update {0}/ set DATA = mscal.stokes(DATA,'circ')".format(time_slice))
                opened_ms.close()

                opened_ms=pt.taql(
                    "update {0}/POLARIZATION set CORR_TYPE=[5,6,7,8]".format(time_slice))
                opened_ms.close()
                self.logger.info("Converted to circular polarization using taql")
            except Exception, exception:
                self.logger.error("Problem applying polarization to ms: {0}".format(
                    time_slice))
                raise exception

    def getConvergedIteration(self, start_time, end_time, start_freq, end_freq):
        # Create fuzzy times and frequencies
        start_time, end_time=self.fuzzyTime(start_time, end_time)
        start_freq, end_freq=self.fuzzyTime(start_freq, end_freq)

        taqlcmd="SELECT STARTTIME, ENDTIME, ITER FROM " + self.tablename + " WHERE STARTTIME>=" + str(start_time) + " AND ENDTIME<=" + str(end_time) + " AND STARTFREQ>=" + str(start_freq) + " AND ENDFREQ<=" + str(end_freq) + " AND LASTITER=TRUE"
        
        result=pt.taql(taqlcmd)              # execute TaQL command
        iteration=result.getcol("ITER")      # get ITER parameter

        return iteration

def uvflux(ms,column,baseline):
  taqlquery="select gstddev(SUMMED) as STDVALS, gmean(SUMMED) as MEANVALS, gcount(SUMMED) as NVALS from (select gmean(mean(0.5*(abs(%s[,0])+abs(%s[,3])))) as SUMMED from %s where (mscal.baseline('%s') and any(FLAG)!=True) GROUP BY TIME)"%(column,column,ms,baseline)
  stats = pt.taql(taqlquery)
  if stats.nrows() > 0:
    meanvals = stats.getcol('MEANVALS')[0]
    nvals = stats.getcol('NVALS')[0]
    stdvals = stats.getcol('STDVALS')[0]/sqrt(nvals)
    print ms,': from',nvals,'time samples, flux is',meanvals,'+/-',stdvals,'(%.2f%% fractional uncertainty)'%((stdvals/meanvals)*100.)
    return meanvals,stdvals
  else:
    print 'Subband %s is totally flagged, no fluxes here'%ms
    return 0., 0.