Python AdmitLogging.AdmitLogging类代码示例

    def setUp(self):
        self.verbose = False
        self.testName = "Utility AdmitLogging Class Unit Test"
        self.logfile = '/tmp/logging_unit_test_%s.log' % os.getpid()
        self.level = 1

        # only need to initialize the logger once since it is a static class
        if not TestAdmitLogging.setup:
            Alogging.init(name="test", logfile=self.logfile, level=Alogging.DEBUG)
            Alogging.addLevelName(15, "TIMING")
            Alogging.addLevelName(16, "REGRSSION")
            Alogging.setLevel(self.level)
            TestAdmitLogging.setup = True

    def set_mask(self, mask):
        """ Method to set the mask item

            Parameters
            ----------
            mask : array like or bool
                The mask to use, must be equal in dimmension to
                the spectral axis, or a single boolean value which
                is applied to all data.

            Returns
            -------
            None

        """
        tempmask = self._mask
        if isinstance(mask, ma.masked_array):
            if len(self._spec) > len(mask) > 1:
                raise
            elif len(mask) == 1:
                self._mask = np.array([mask.data[0]] * len(self._spec))
            else:
                self._mask = np.array(mask.data)
        elif isinstance(mask, list) or isinstance(mask, np.ndarray):
            if len(self._spec) > len(mask) > 1:
                raise
            elif len(mask) == 1:
                self._mask = np.array([mask[0]] * len(self._spec))
            else:
                self._mask = np.array(mask)
        if self.integritycheck():
            return
        logging.warning("  Mask not applied")
        self._mask = tempmask

    def mask_ge(self, limit, axis="spec"):
        """ Method to mask any data less than or equal to the given value.
            Any axis can be used ("spec", "freq", "chans") as
            the basis for the masking.

            Parameters
            ----------
            limit : float or int
                The value which all data, of the given axis,
                greater than or equal to, will be masked.

            axis : str
                The axis which is used to determine the flags.
                Default: "spec"

            Returns
            -------
            None

        """
        if not self.integritycheck():
            logging.warning("  Masking operation not performed.")
            return
        mask = getattr(self, "_" + axis, None) >= limit
        self._mask = np.logical_or(self._mask, mask)

    def mask_outside(self, limit1, limit2, axis="spec"):
        """ Method to mask any data less than or greater than the given values.
            Any axis can be used ("spec", "freq", "chans") as
            the basis for the masking.

            Parameters
            ----------
            limit1 : float or int
                The value below which all data of the given axis
                will be masked.

            limit2 : float or int
                The value above which all data of the given axis
                will be masked.

            axis : str
                The axis which is used to determine the flags.
                Default: "spec"

            Returns
            -------
            None

        """
        if not self.integritycheck():
            logging.warning("  Masking operation not performed.")
            return
        mask1 = getattr(self, "_" + axis, None) < limit1
        mask2 = limit2 < getattr(self, "_" + axis, None)
        mask = np.logical_and(mask1, mask2)
        self._mask = np.logical_or(self._mask, mask)

 def end(self):
     t0 = self.init
     t1 = self.time()
     dt = t1 - t0
     if self.report:
         logging.timing("%s END " % self.label + str(dt))
     return dt

    def test_warning(self):
        msg = "unit_test_warning_message"
        Alogging.warning(msg)
  
        found = False
        r = open(self.logfile, 'r')
        for line in r.readlines():
            if msg in line:
                if(self.verbose):
                    print "\nFound message >", line
 
                found = True
                r.close()
                break
 
        # since this is the last test case, we now close off the logging
        Alogging.shutdown()

        try:
            if os.path.exists(self.logfile):
                os.remove(self.logfile)

        except RuntimeError:
            pass
 
        self.assertTrue(found)

    def addBDPtoAT(self, bdp):
        """ Method to add a BDP to an AT. The AT is not specified, but the
            _taskid attribute of the BDP is used to identify the necessary AT.

            Parameters
            ----------
            bdp : BDP
                Any valid BDP, to be added to an existing AT.

            Returns
            -------
            None

        """
        found = False
        cp = copy.deepcopy(bdp)
        # find the AT we need
        for at in self.tasks:
            # see if the ID's match
            if at._taskid == bdp._taskid:
                found = True
                # set the base directory of the BDP
                cp.baseDir(at.baseDir())
                # add it to the correct slot
                at._bdp_out[at._bdp_out_map.index(cp._uid)] = cp
                break
        if not found:
            logging.info("##### Found orphaned BDP with type %s in file %s" % \
                (bdp._type, bdp.xmlFile))

    def checkAll(self):
        """ Method to check the dtd structure to see if all expected
            nodes were found.

            Parameters
            ----------
            None

            Returns
            -------
            Boolean, whether or not all nodes were found

        """
        #pp.pprint(self.entities)
        for i in self.entities:
            if not self.entities[i]["found"]:
                print self.xmlFile
                logging.info(str(i) + " not found")
                return False
            for a in self.entities[i]["attrib"]:
                if not self.entities[i]["attrib"][a]["found"]:
                    print "2",self.xmlFile
                    logging.info(str(i) + " " + str(a) + " not found")
                    return False
        return True

def admit_root(path = None):
    """ Return the root directory of the ADMIT environment

        Typically in ADMIT/etc there are files needed by certain functions
        and other TBD locations.
        For now we are using getenv(), but in deployment this may not
        be the case.

        Parameters
        ----------
        path : string
           Optional path appended to the ADMIT root

        Returns
        -------
        String containing the absolute address of the admit root directory,
        with the optional path appended
    """
    global _admit_root
    if _admit_root != None:  return _admit_root

    # try the old style developer environment first
    _admit_root = os.getenv("ADMIT")
    if _admit_root == None:
        # try the dumb generic way;  is that safe to reference from __file__ ???
        _admit_root = version.__file__.rsplit('/',2)[0]
        if _admit_root[0] != '/':
            logging.warning("ADMIT_ROOT is a relative address")
            # @tdodo shouldn't that be a fatal error
    # 
    print "_ADMIT_ROOT=",_admit_root
    if path == None:
        return _admit_root
    return _admit_root + '/' + path

    def get_mem(self):
        """ Read memory usage info from /proc/pid/status
            Return Virtual and Resident memory size in MBytes.
        """
        global ostype
        
        if ostype == None:
            ostype = os.uname()[0].lower()
            logging.info("OSTYPE: %s" % ostype)
            
        scale = {'MB': 1024.0}
        lines = []
        try:
            if ostype == 'linux':
                proc_status = '/proc/%d/status' % os.getpid()          # linux only
                # open pseudo file  /proc/<pid>/status
                t = open(proc_status)
                # get value from line e.g. 'VmRSS:  9999  kB\n'
                for it in t.readlines():
                    if 'VmSize' in it or 'VmRSS' in it :
                        lines.append(it)
                t.close()
            else:
                proc = subprocess.Popen(['ps','-o', 'rss', '-o', 'vsz', '-o','pid', '-p',str(os.getpid())],stdout=subprocess.PIPE)
                proc_output = proc.communicate()[0].split('\n') 
                proc_output_memory = proc_output[1]
                proc_output_memory = proc_output_memory.split()
                
                phys_mem = int(proc_output_memory[0])/1204 # to MB 
                virtual_mem = int(proc_output_memory[1])/1024 
                
        except (IOError, OSError):
            if self.report:
                logging.timing(self.label + " Error: cannot read memory usage information.")

            return np.array([])

        # parse the two lines
    
        mem = {}
        if(ostype != 'darwin'):
            for line in lines:
                words = line.strip().split()
            #print words[0], '===', words[1], '===', words[2]
                
            # get rid of the tailing ':'
                key = words[0][:-1]

            # convert from KB to MB
                scaled = float(words[1]) / scale['MB']
                mem[key] = scaled
        else:
            mem['VmSize'] = virtual_mem
            mem['VmRSS']  = phys_mem


        return np.array([mem['VmSize'], mem['VmRSS']])

 def __init__(self, upper=True):
     self.version = "27-apr-2016"
     if have_ADMIT:
         self.table = utils.admit_root() + "/etc/vlsr.tab"
         self.cat = read_vlsr(self.table,upper)
         logging.debug("VLSR: %s, found %d entries" % (self.table,len(self.cat)))
     else:
         logging.warning("VLSR: Warning, no ADMIT, empty catalogue")
         self.cat = {}

 def __init__(self, label=".", report=True):
     self.start = self.time()
     self.init = self.start
     self.label = label
     self.report = report
     self.dtimes = []
     dt = self.init - self.init
     if self.report:
         logging.timing("%s ADMIT " % self.label + str(self.start))
         logging.timing("%s BEGIN " % self.label + str(dt))

def peakstats(image, freq, sigma, nsigma, minchan, maxgap, psample, peakfit = False):
    """ Go through a cube and find peaks in the spectral dimension

    It will gather a table of <peak>,<freq>,<sigma> which can be
    optionally used for plotting
    """
    if psample < 0: return
    cutoff = nsigma * sigma
    madata = casautil.getdata(image)
    data   = madata.data
    shape  = data.shape
    logging.debug("peakstats: shape=%s cutoff=%g" % (str(shape),cutoff))
    #print "DATA SHAPE:",shape
    #print "cutoff=",cutoff
    nx = shape[0]
    ny = shape[1]
    nz = shape[2]
    chan = np.arange(nz)
    # prepare the segment finder
    # we now have an array data[nx,ny,nz]
    sum = 0.0
    pval = []
    mval = []
    wval = []
    for x in range(0,nx,psample):
        for y in range(0,ny,psample):
            s0    = data[x,y,:]
            spec  = ma.masked_invalid(s0)
            sum += spec.sum()
            # using abs=True is a bit counter intuitive, but a patch to deal with the confusion in
            # ADMITSegmentFinder w.r.t abs usage
            asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=True)
            #asf = ADMITSegmentFinder(pmin=nsigma, minchan=minchan, maxgap=maxgap, freq=freq, spec=spec, abs=False)
            f = asf.line_segments(spec, nsigma*sigma)
            for s in f:
                if False:
                    for i in range(s[0],s[1]+1):
                        print "# ",x,y,i,spec[i]
                ## area preserving and peak are correlated, 18% difference
                ## fitgauss1Dm was about 5"
                ## with fitgauss1D was about 30", and still bad fits
                par      = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], True)           # peak from max
                #par      = utils.fitgauss1Dm(chan[s[0]:s[1]+1], spec[s[0]:s[1]+1], False)       # peak from area preserving
                if peakfit:
                    (par,cov) = utils.fitgauss1D (chan[s[0]:s[1]+1], spec[s[0]:s[1]+1],par)
                #print "FIND:  ",x,y,s,cutoff,0.0,0.0,par[0],par[1],par[2],s[1]-s[0]+1
                pval.append(par[0])
                mval.append(par[1])
                wval.append(par[2])
    #print "SUM:",sum
    return (np.array(pval),np.array(mval),np.array(wval))

    def test_debug(self):
        msg = "unit_test_debug_message"
        Alogging.debug(msg)
 
        found = False
        r = open(self.logfile, 'r')
        for line in r.readlines():
            if msg in line:
                if(self.verbose):
                    print "\nFound message > ", line
                found = True
                r.close()
                break
 
        self.assertTrue(found)

    def tag(self, mytag):

        t0 = self.start
        t1 = self.time()
        dt = t1 - t0

        # get memory usage (Virtual and Resident) info
        mem = self.get_mem()
        if mem.size != 0 :
            dt = np.append(dt, mem)

        self.dtimes.append((mytag, dt))
        self.start = t1
        if self.report:
            logging.timing("%s " % self.label + mytag + "  " + str(dt))
        return dt