Python db.CatalogDBObject类代码示例

 def testDbObj(self):
     mystars = CatalogDBObject.from_objid('testCatalogDBObjectTeststars')
     mygals = CatalogDBObject.from_objid('testCatalogDBObjectTestgals')
     result = mystars.query_columns(obs_metadata=self.obsMd)
     tu.writeResult(result, "/dev/null")
     result = mygals.query_columns(obs_metadata=self.obsMd)
     tu.writeResult(result, "/dev/null")

def examplePhoSimCatalogs():
    """
    This method outputs several phoSim input files consisting of different
    types of objects(stars, galaxy bulges, galaxy disks, and AGNs)

    The files created are
    catalog_test_msstars.dat
    catalog_test_galaxyDisk.dat
    catalog_test_galaxyBulge.dat
    catalog_test_galaxyAgn.dat

    (versions are also created that end in _chunked.dat; these should have
    the same contents)

    """
    obsMD = bcm.OpSim3_61DBObject()
    obs_metadata = obsMD.getObservationMetaData(88544919, 0.1, makeCircBounds=True)
    objectDict = {}
    objectDict['testStars'] = {'dbobj':CatalogDBObject.from_objid('msstars'),
                               'constraint':None,
                               'filetype':'phoSim_catalog_POINT',
                               'obsMetadata':obs_metadata}
    objectDict['testGalaxyBulge'] = {'dbobj':CatalogDBObject.from_objid('galaxyBulge'),
                               'constraint':"mass_bulge > 1. and sedname_bulge is not NULL",
                               'filetype':'phoSim_catalog_SERSIC2D',
                               'obsMetadata':obs_metadata}
    objectDict['testGalaxyDisk'] = {'dbobj':CatalogDBObject.from_objid('galaxyDisk'),
                               'constraint':"DiskLSSTg < 20. and sedname_disk is not NULL",
                               'filetype':'phoSim_catalog_SERSIC2D',
                               'obsMetadata':obs_metadata}
    objectDict['testGalaxyAgn'] = {'dbobj':CatalogDBObject.from_objid('galaxyAgn'),
                               'constraint':"sedname_agn is not NULL",
                               'filetype':'phoSim_catalog_ZPOINT',
                               'obsMetadata':obs_metadata}

    for objKey in objectDict.keys():
        dbobj = objectDict[objKey]['dbobj']
        t = dbobj.getCatalog(objectDict[objKey]['filetype'],
                             obs_metadata=objectDict[objKey]['obsMetadata'],
                             constraint=objectDict[objKey]['constraint'])

        print
        print "These are the required columns from the database:"
        print t.db_required_columns()
        print
        print "These are the columns that will be output to the file:"
        print t.column_outputs
        print

        filename = 'catalog_test_%s.dat'%(dbobj.objid)
        print "querying and writing catalog to %s:" % filename
        t.write_catalog(filename)
        filename = 'catalog_test_%s_chunked.dat'%(dbobj.objid)
        t.write_catalog(filename, chunk_size=10)
        print " - finished"

    def testHybridVariability(self):
        """
        Test that we can generate a catalog which inherits from multiple variability mixins
        (in this case, TestVariability and VariabilityStars).  This is to make sure that
        the register_method and register_class decorators do not mangle inheritance of
        methods from mixins.
        """
        makeHybridTable()
        myDB = CatalogDBObject.from_objid('hybridTest')
        myCatalog = myDB.getCatalog('testVariabilityCatalog', obs_metadata=self.obs_metadata)
        myCatalog.write_catalog('hybridTestCatalog.dat', chunk_size=1000)

        if os.path.exists('hybridTestCatalog.dat'):
            os.unlink('hybridTestCatalog.dat')

        # make sure order of mixin inheritance does not matter
        myCatalog = myDB.getCatalog('otherVariabilityCatalog', obs_metadata=self.obs_metadata)
        myCatalog.write_catalog('hybridTestCatalog.dat', chunk_size=1000)

        if os.path.exists('hybridTestCatalog.dat'):
            os.unlink('hybridTestCatalog.dat')

        # make sure that, if a catalog does not contain a variability method,
        # an error is thrown; verify that it contains the correct error message
        myCatalog = myDB.getCatalog('stellarVariabilityCatalog', obs_metadata=self.obs_metadata)
        with self.assertRaises(RuntimeError) as context:
            myCatalog.write_catalog('hybridTestCatalog.dat')

        expectedMessage = "Your InstanceCatalog does not contain a variability method"
        expectedMessage += " corresponding to 'testVar'"
        self.assertTrue(context.exception.message==expectedMessage)

        if os.path.exists('hybridTestCatalog.dat'):
            os.unlink('hybridTestCatalog.dat')

    def testNonsenseSelectOnlySomeColumns(self):
        """
        Test a query performed only a subset of the available columns
        """
        myNonsense = CatalogDBObject.from_objid('Nonsense')

        mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseMag']

        query = myNonsense.query_columns(colnames=mycolumns, constraint = 'ra < 45.', chunk_size=100)

        goodPoints = []

        for chunk in query:
            for row in chunk:
                self.assertTrue(row[1]<45.0)

                dex = numpy.where(self.baselineData['id'] == row[0])[0][0]

                goodPoints.append(row[0])

                self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3)
                self.assertAlmostEqual(self.baselineData['mag'][dex],row[2],3)


        for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
            self.assertTrue(entry[1]>45.0)

        def testCanBeNull(self):
            """
            Test to make sure that we can still write all rows to catalogs,
            even those with null values in key columns
            """
            dbobj = CatalogDBObject.from_objid('cannotBeNull')
            cat = dbobj.getCatalog('canBeNull')
            fileName = 'canBeNullTestFile.txt'
            cat.write_catalog(fileName)
            dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64),
                                 ('n4',(str,40)), ('n5',(unicode,40))])
            testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',')

            for i in range(len(self.baselineOutput)):
                #make sure that all of the rows in self.baselineOutput are represented in
                #testData
                for (k,xx) in enumerate(self.baselineOutput[i]):
                    if k<4:
                        if not numpy.isnan(xx):
                            self.assertAlmostEqual(xx,testData[i][k], 3)
                        else:
                            self.assertTrue(numpy.isnan(testData[i][k]))
                    else:
                        msg = '%s is not %s' % (xx,testData[i][k])
                        self.assertEqual(xx.strip(),testData[i][k].strip(),msg=msg)

            self.assertEqual(i,99)
            self.assertEqual(len(testData), len(self.baselineOutput))

            if os.path.exists(fileName):
                os.unlink(fileName)

def exampleAirmass(airmass,ra = 0.0, dec = 0.0, tol = 10.0, radiusDeg = 0.1,
            makeBoxBounds=False, makeCircBounds=True):
    """
    This method will output a catalog of stars based on an OpSim pointing with
    a specific airmass.  It searches OpSim for pointings with the specified airmass
    and RA, Dec within a box bounded by tol (in degrees).  It creates observation meta
    data out of the first pointing found and uses that to construct the catalog.

    The catalog is output to stars_airmass_test.dat
    """

    obsMD=bcm.OpSim3_61DBObject()

    #The code below will query the OpSim data base object created above.
    #The query will be based on a box in RA, Dec and a specific airmass value

    airmassConstraint = "airmass="+str(airmass) #an SQL constraint that the airmass must be equal to
                                                #the passed value

    skyBounds = SpatialBounds.getSpatialBounds('box', ra, dec, tol) #bounds on region of sky

    query = obsMD.executeConstrainedQuery(skyBounds, constraint=airmassConstraint)

    #convert q into observation meta data for use in a catalog
    obsMetaData = obsMD.getObservationMetaData(query['Opsim_obshistid'][0],radiusDeg,makeBoxBounds=makeBoxBounds,
                   makeCircBounds=makeCircBounds)

    #create and output a reference catalog of stars based on our query to opSim
    dbobj = CatalogDBObject.from_objid('allstars')
    catalog = dbobj.getCatalog('ref_catalog_star', obs_metadata = obsMetaData)
    catalog.write_catalog('stars_airmass_test.dat')

        def testCanBeNull(self):
            """
            Test to make sure that we can still write all rows to catalogs,
            even those with null values in key columns
            """
            baselineOutput = createCannotBeNullTestDB()
            dbobj = CatalogDBObject.from_objid('cannotBeNull')
            cat = dbobj.getCatalog('canBeNull')
            fileName = 'canBeNullTestFile.txt'
            cat.write_catalog(fileName)
            dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64)])
            testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',')

            for i in range(len(baselineOutput)):
                if not numpy.isnan(baselineOutput['n2'][i]):
                    for (k,xx) in enumerate(baselineOutput[i]):
                        if not numpy.isnan(xx):
                            self.assertAlmostEqual(xx,testData[i][k],3)
                        else:
                            self.assertTrue(numpy.isnan(testData[i][k]))

            self.assertEqual(i,99)

            if os.path.exists(fileName):
                os.unlink(fileName)

    def testEb(self):
        makeEbTable()
        myDB = CatalogDBObject.from_objid('ebTest')
        myCatalog = myDB.getCatalog('stellarVariabilityCatalog', obs_metadata=self.obs_metadata)
        myCatalog.write_catalog('ebTestCatalog.dat', chunk_size=1000)

        if os.path.exists('ebTestCatalog.dat'):
            os.unlink('ebTestCatalog.dat')

        def testCannotBeNull(self):
            """
            Test to make sure that the code for filtering out rows with null values
            in key rowss works.
            """

            #each of these classes flags a different column with a different datatype as cannot_be_null
            availableCatalogs = [floatCannotBeNullCatalog, strCannotBeNullCatalog, unicodeCannotBeNullCatalog]
            dbobj = CatalogDBObject.from_objid('cannotBeNull')

            for catClass in availableCatalogs:
                cat = catClass(dbobj)
                fileName = 'cannotBeNullTestFile.txt'
                cat.write_catalog(fileName)
                dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64),
                                     ('n4',(str,40)), ('n5',(unicode,40))])
                testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',')

                j = 0 #a counter to keep track of the rows read in from the catalog

                for i in range(len(self.baselineOutput)):

                    #self.baselineOutput contains all of the rows from the dbobj
                    #first, we must assess whether or not the row we are currently
                    #testing would, in fact, pass the cannot_be_null test
                    validLine = True
                    if isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],str) or \
                       isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],unicode):

                        if self.baselineOutput[cat.cannot_be_null[0]][i].strip().lower() == 'none':
                            validLine = False
                    else:
                        if numpy.isnan(self.baselineOutput[cat.cannot_be_null[0]][i]):
                            validLine = False

                    if validLine:
                        #if the row in self.baslineOutput should be in the catalog, we now check
                        #that baseline and testData agree on column values (there are some gymnastics
                        #here because you cannot do an == on NaN's
                        for (k,xx) in enumerate(self.baselineOutput[i]):
                            if k<4:
                                if not numpy.isnan(xx):
                                    msg = 'k: %d -- %s %s -- %s' % (k,str(xx),str(testData[j][k]),cat.cannot_be_null)
                                    self.assertAlmostEqual(xx, testData[j][k],3, msg=msg)
                                else:
                                    self.assertTrue(numpy.isnan(testData[j][k]))
                            else:
                                msg = '%s (%s) is not %s (%s)' % (xx,type(xx),testData[j][k],type(testData[j][k]))
                                self.assertEqual(xx.strip(),testData[j][k].strip(), msg=msg)
                        j+=1

                self.assertEqual(i,99) #make sure that we tested all of the baseline rows
                self.assertEqual(j,len(testData)) #make sure that we tested all of the testData rows
                msg = '%d >= %d' % (j,i)
                self.assertTrue(j<i, msg=msg) #make sure that some rows did not make it into the catalog

            if os.path.exists(fileName):
                os.unlink(fileName)

 def __init__(self, objid, phosimCatalogObject):
     while True:
     # This loop is a workaround for UW catsim db connection intermittency.
         try:
             self.db_obj = CatalogDBObject.from_objid(objid)
             break
         except RuntimeError:
             continue
     self.cat_obj = phosimCatalogObject

    def testAgn(self):

        makeAgnTable()
        myDB = CatalogDBObject.from_objid('agnTest')
        myCatalog = myDB.getCatalog('galaxyVariabilityCatalog', obs_metadata=self.obs_metadata)
        myCatalog.write_catalog('agnTestCatalog.dat', chunk_size=1000)

        if os.path.exists('agnTestCatalog.dat'):
            os.unlink('agnTestCatalog.dat')

 def testArbitraryQuery(self):
     """
     Test method to directly execute an arbitrary SQL query (inherited from DBObject class)
     """
     myNonsense = CatalogDBObject.from_objid('Nonsense')
     query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id'
     results = myNonsense.execute_arbitrary(query)
     self.assertEqual(len(results),1250)
     for row in results:
         self.assertEqual(row[0],row[2])
         self.assertAlmostEqual(row[1],0.5*row[3],6)

    def testNonsenseArbitraryConstraints(self):
        """
        Test a query with a user-specified constraint on the magnitude column
        """

        myNonsense = CatalogDBObject.from_objid('Nonsense')

        raMin = 50.0
        raMax = 150.0
        decMax = 30.0
        decMin = -20.0
        raCenter=0.5*(raMin+raMax)
        decCenter=0.5*(decMin+decMax)

        mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseDecJ2000','NonsenseMag']

        boxObsMd = ObservationMetaData(boundType='box',unrefractedRA=raCenter,unrefractedDec=decCenter,
                    boundLength=numpy.array([0.5*(raMax-raMin),0.5*(decMax-decMin)]), mjd=52000.,bandpassName='r')

        boxQuery = myNonsense.query_columns(colnames = mycolumns,
                      obs_metadata=boxObsMd, chunk_size=100, constraint = 'mag > 11.0')

        raMin = numpy.radians(raMin)
        raMax = numpy.radians(raMax)
        decMin = numpy.radians(decMin)
        decMax = numpy.radians(decMax)

        goodPoints = []

        for chunk in boxQuery:
            for row in chunk:

                self.assertTrue(row[1]<raMax)
                self.assertTrue(row[1]>raMin)
                self.assertTrue(row[2]<decMax)
                self.assertTrue(row[2]>decMin)
                self.assertTrue(row[3]>11.0)

                dex = numpy.where(self.baselineData['id'] == row[0])[0][0]

                #keep a list of the points returned by the query
                goodPoints.append(row[0])

                self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3)
                self.assertAlmostEqual(numpy.radians(self.baselineData['dec'][dex]),row[2],3)
                self.assertAlmostEqual(self.baselineData['mag'][dex],row[3],3)

        for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
            #make sure that the points not returned by the query did, in fact, violate one of the
            #constraints of the query (either the box bound or the magnitude cut off)
            switch = (entry[1] > raMax or entry[1] < raMin or entry[2] >decMax or entry[2] < decMin or entry[3]<11.0)

            self.assertTrue(switch)

    def testNonsenseBoxConstraints(self):
        """
        Test that a query performed on a box bound gets all of the points (and only all of the
        points) inside that box bound.
        """

        myNonsense = CatalogDBObject.from_objid('Nonsense')

        raMin = 50.0
        raMax = 150.0
        decMax = 30.0
        decMin = -20.0

        raCenter = 0.5*(raMin+raMax)
        decCenter = 0.5*(decMin+decMax)

        mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseDecJ2000','NonsenseMag']

        boxObsMd = ObservationMetaData(boundType='box',unrefractedDec=decCenter, unrefractedRA=raCenter,
                   boundLength=numpy.array([0.5*(raMax-raMin),0.5*(decMax-decMin)]),mjd=52000.,bandpassName='r')

        boxQuery = myNonsense.query_columns(obs_metadata=boxObsMd, chunk_size=100, colnames=mycolumns)

        raMin = numpy.radians(raMin)
        raMax = numpy.radians(raMax)
        decMin = numpy.radians(decMin)
        decMax = numpy.radians(decMax)

        goodPoints = []

        for chunk in boxQuery:
            for row in chunk:
                self.assertTrue(row[1]<raMax)
                self.assertTrue(row[1]>raMin)
                self.assertTrue(row[2]<decMax)
                self.assertTrue(row[2]>decMin)

                dex = numpy.where(self.baselineData['id'] == row[0])[0][0]

                #keep a list of which points were returned by teh query
                goodPoints.append(row[0])

                self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3)
                self.assertAlmostEqual(numpy.radians(self.baselineData['dec'][dex]),row[2],3)
                self.assertAlmostEqual(self.baselineData['mag'][dex],row[3],3)

        for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
            #make sure that the points not returned by the query are, in fact, outside of the
            #box bound

            switch = (entry[1] > raMax or entry[1] < raMin or entry[2] >decMax or entry[2] < decMin)
            self.assertTrue(switch)

    def testRealQueryConstraints(self):
        mystars = CatalogDBObject.from_objid('testCatalogDBObjectTeststars')
        mycolumns = ['id','raJ2000','decJ2000','umag','gmag','rmag','imag','zmag','ymag']

        #recall that ra and dec are stored in degrees in the data base
        myquery = mystars.query_columns(colnames = mycolumns,
                                        constraint = 'ra < 90. and ra > 45.')

        tol=1.0e-3
        for chunk in myquery:
            for star in chunk:
                self.assertTrue(numpy.degrees(star[1])<90.0+tol)
                self.assertTrue(numpy.degrees(star[1])>45.0-tol)