Python Database.commitOne方法代码示例

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class CouchSink(object):
    """
    Alert sink for pushing alerts to a couch database.

    """

    def __init__(self, config):
        self.config = config
        logging.info("Instantiating ...")
        # test if the configured database does not exist, create it
        server = CouchServer(self.config.url)
        databases = server.listDatabases()
        if self.config.database not in databases:
            logging.warn(
                "'%s' database does not exist on %s, creating it ..." % (self.config.database, self.config.url)
            )
            server.createDatabase(self.config.database)
            logging.warn("Created.")
        logging.info("'%s' database exists on %s" % (self.config.database, self.config.url))
        self.database = Database(self.config.database, self.config.url)
        logging.info("Initialized.")

    def send(self, alerts):
        """
        Handle list of alerts.

        """
        retVals = []
        for a in alerts:
            doc = Document(None, a)
            retVal = self.database.commitOne(doc)
            retVals.append(retVal)
        logging.debug("Stored %s alerts to CouchDB, retVals: %s" % (len(alerts), retVals))
        return retVals

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class ReproducibleSeedingTests(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
        self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache")
        self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl)

        self.documentId = None

    def tearDown(self):
        self.testInit.tearDownCouch()
        return


    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
                                     CouchDBName = self.testInit.couchDbName,
                                     ConfigCacheDoc = document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
        self.failUnless(seed1 != None)

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
def update_software(config_file):
    """
    Functions retrieves CMSSW versions and scramarchs from CMS tag collector.
    
    """
    config = loadConfigurationFile(config_file)
    # source of the data
    tag_collector_url = config.views.data.tag_collector_url
    # store the data into CouchDB auxiliary database under "software" document
    couch_host = config.views.data.couch_host
    reqmgr_aux_db = config.views.data.couch_reqmgr_aux_db
    
    # get data from tag collector
    all_archs_and_versions = _get_all_scramarchs_and_versions(tag_collector_url)
    if not all_archs_and_versions:
        return
    
    # get data already stored in CouchDB    
    couchdb = Database(dbname=reqmgr_aux_db, url=couch_host)
    try:
        sw_already_stored = couchdb.document("software")
        del sw_already_stored["_id"]
        del sw_already_stored["_rev"]
    except CouchNotFoundError:
        logging.error("Document id software, does not exist, creating it ...")
        doc = Document(id="software", inputDict=all_archs_and_versions)
        couchdb.commitOne(doc)
        return
    
    # now compare recent data from tag collector and what we already have stored
    # sorting is necessary
    if sorted(all_archs_and_versions) != sorted(sw_already_stored):
        logging.debug("ScramArch/CMSSW releases changed, updating software document ...")
        doc = Document(id="software", inputDict=all_archs_and_versions)
        couchdb.commitOne(doc)
    """

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class CouchSink(object):
    """
    Alert sink for pushing alerts to a couch database.
    
    """     
    def __init__(self, config):
        self.config = config
        self.database = Database(self.config.database, self.config.url)
        
        
    def send(self, alerts):
        """
        Handle list of alerts.
        
        """
        retVals = []
        for a in alerts:
            doc = Document(None, a)
            retVal = self.database.commitOne(doc)
            retVals.append(retVal)
        return retVals

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class TestDQISResult(unittest.TestCase):
    DB_NAME = 'dqis_test'
    DB_URL = 'localhost:5984'

    def setUp(self):        
        couch = CouchServer(dburl=self.DB_URL)
        if self.DB_NAME in couch.listDatabases():
            couch.deleteDatabase(self.DB_NAME)
        
        cdb = couch.connectDatabase(self.DB_NAME)

        #for dq_t in test_data.demo_data:
        #    cdb.queue(dq_t)
        
        cdb.commit()
        
        self.db = Database(dbname=self.DB_NAME)
        
    
    def test_init(self):        
        #self.assertEqual(1,2)
        pass

    def test_save_and_delete(self):
        #Shoud document get revision number after save?
        #Document can not be saved and then deleted. Because save returns not a DQISResult object!
        
        #Tests document saving 
        document = {"_id": "abc", "test":"data"}
        r = API.DQISResult(dqis_db = self.db, dict = document)
        all_docs_count_before = len(self.db.allDocs()['rows'])
        r.save()
        all_docs_count_after_insert = len(self.db.allDocs()['rows'])        
        
        self.assertEqual(all_docs_count_before +1, all_docs_count_after_insert)
        
        
        #Test delete
        doc = self.db.document("abc")
        r = API.DQISResult(dict=doc, dqis_db = self.db)
        self.assertEqual(doc["test"], "data")
        r.delete()
        self.db.commitOne(r)
        all_docs_count_after_deleting = len(self.db.allDocs()['rows']) 
        self.assertEqual(all_docs_count_before, all_docs_count_after_deleting )
        
    def test_savable(self):
        #Does ID has to raise exception
        rez = API.DQISResult(dict = {'_id': "123"})._require_savable()
        self.assertEqual(rez, None)
        self.assertRaises(DQISResultNotSavable, 
                    API.DQISResult(dict = {'id': "123"})._require_savable )
        self.assertRaises(DQISResultNotSavable, 
                    API.DQISResult(dict = {'abc': "123"})._require_savable )
        
    def test_find_id(self): #similar to test_savable
        self.assertEqual(DQISResult()._find_id(), "")
        self.assertEqual(DQISResult(dict = {'id': "123"})._find_id(), "123")
        self.assertEqual(DQISResult(dict = {'_id': "123"})._find_id(), "123") 
        
    def test_find_id(self):
        id1 = API.DQISResult()._find_id()
        id2 = API.DQISResult(dict = {'id': "123"})._find_id()
        id3 = API.DQISResult(dict = {'_id': "abc"})._find_id()
        self.assertEqual(id1, "")
        self.assertEqual(id2, '123')
        self.assertEqual(id3, 'abc')
        
    def test_require_saveable(self):
        dr1 = API.DQISResult()._require_savable
        #dr2 = API.DQISResult(dict = {'_id': "123"})._require_savable
        self.assertRaises(DQISResultNotSavable, dr1)
        #self.assertEqual(None, dr2())
        
    def test_save_to_queue(self):
        r = DQISResult(dqis_db = Database(), dict = {"_id": "abc"})
        queue_size_before = len(r.dqis_db._queue)
        r.saveToQueue()
        queue_size_after = len(r.dqis_db._queue) 
        self.assertEqual(queue_size_before, 0)
        self.assertEqual(queue_size_after, 1)
        r.dqis_db._reset_queue()
        
        
    def test_require_db(self):
        f = DQISResult()._require_db_connection
        self.assertRaises(DatabaseNotSetException, f)  
        
        f = DQISResult(dqis_db = "dqis_db")._require_db_connection
        self.assertRaises(DatabaseNotSetException, f)  
        
        f = DQISResult(dqis_db = Database())._require_db_connection
        self.assertEqual(None, f())



    def test_get_document(self):
        doc_id = '100215-0-38bc1d29bd22844103e86f9a000500e2' 
        r = API.DQISResult(API.Database(dbname="dqis"))
        r['id'] = doc_id
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]

#.........这里部分代码省略.........
        opts = {} 
        
        # default the composite key search
        #if '[' in val and ']' in val:
        if val.startswith('[') and val.endswith(']'):
            if op == '==':
                try:                    
                    e=ast.literal_eval(val)
                    opts['key'] = e
                except:
                    opts['key'] = val
            return opts
        
        # handle alphanumeric key ranges
        num_flag = False
        if is_number(val):
            num_flag = True
            kval = float(val)
        else:
            kval = val.decode('ascii')
        if '>' in op:
            if '=' in op:
                opts['startkey']=kval
            else:
                if num_flag:
                    opts['startkey']=kval+1
                else:
                    opts['startkey']=kval
            if num_flag:
                opts['endkey']=99999999 # assume its numeric
            else:
                opts['endkey']=kval+u'\u9999'
        elif '<' in op:
            if '=' in op:
                opts['endkey']=kval
            else:
                if num_flag:
                    opts['endkey']=kval-1
                else:
                    opts['endkey']=kval
            if num_flag:
                opts['startkey']=-99999999
            else:
                opts['startkey']=''
                
        elif '==' == op:
            opts['key']=kval
        elif '~=' == op:
            if kval[-1] == '*':
                opts['startkey']=kval[:len(kval)-1]
                opts['endkey']=kval[:len(kval)-1]+u'\u9999'#'99999999'#'\u9999'
        return opts
            
  
    def save_all(self,  docs=[]):
        if not docs:
            return False
        for doc in docs:
            self.db.queue(doc)
        try:
            self.db.commit()
            return True
        except Exception as ex:
            self.logger.error('Could not commit changes to database. Reason: %s' % (ex)) 
            return False

    def save(self, doc={}):
        if not doc:
            self.logger.error('Tried to save empty document.', level='warning')
            return False


	# TODO: Check if an object exists in the database and fail.

        #if '_id' in doc:
        #    self.logger.log('Using user-defined id: %s' % (doc['_id']))
        #if self.__document_exists(doc):
        #    self.logger.error('Failed to update document: %s' % (json.dumps(doc)))
        #    return False

        try:
            #self.logger.error('Document is %s %s'%(doc['_id'],doc))
            #self.logger.error(self.db.commitOne(doc))
            ## this is a change I just made (23/05/2013 13:31) because of the return value of update should be True/False
            saved = self.db.commitOne(doc)
            if 'error' in saved[0]:
                self.logger.error('Commit One says : %s'%(saved))
                return False
            else:
                return True
        except Exception as ex:
            self.logger.error('Could not commit changes to database. Reason: %s' % (ex))
            return False

    def count(self):
        try:
            return len(self.db.allDocs()) 
        except Exception as ex:
            self.logger.error('Could not count documents in database. Reason: %s' % (ex))
            return -1 

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
def swapLocations(options):
    #Initialize stuff
    phedexAPI = PhEDEx({'cachepath' : options.cachepath})
    acdcCouch = Database('wmagent_acdc', options.acdcUrl)

    #Let's get the IDs of the ACDC documents for the task/request/group/user
    array = [options.group, options.user, options.request, options.task]
    result = acdcCouch.loadView('ACDC', 'owner_coll_fileset_docs', {'reduce' : False}, [array])

    documentsIDs = [x['id'] for x in result['rows']]

    #Load the map file saying what we want to change of location
    mapFile = open(options.map, 'r')
    locationMap = json.load(mapFile)
    mapFile.close()

    #Go through the documents
    for docID in documentsIDs:
        doc = acdcCouch.document(docID)

        #Are we going to change this doc? Better back it up
        if options.change:
            backupFile = os.open(os.path.join(options.backup, "%s.bkp" % doc["_id"]), 'w')
            json.dump(doc, backupFile)
            backupFile.close()

        #Go through the files
        files = doc["files"]
        for inputFile in files:

            #Use PhEDEx API to get site based on the SE
            #Then map that to the desired target
            se = files[inputFile]["locations"][0]
            siteLocation = phedexAPI.getBestNodeName(se)
            targetLocation = locationMap.get(siteLocation, siteLocation)

            if siteLocation == targetLocation:
                #Nothing to do with this one, move on
                continue

            if not options.change:
                #No changes, then give the commands to move the files
                #Get the PFN for both the current location and the target location
                pfnDict = phedexAPI.getPFN(siteLocation, inputFile)
                inputPfn = pfnDict[(siteLocation, inputFile)]
                pfnDict = phedexAPI.getPFN(targetLocation, inputFile)
                targetPfn = pfnDict[(targetLocation, inputFile)]

                #Print it to stdout
                print "lcg-cp -D srmv2 -b %s %s" % (inputPfn, targetPfn)

            else:
                #This is changes time, let's move the stuff
                targetSE = phedexAPI.getNodeSE(targetLocation)
                files[inputFile]["locations"][0] = targetSE
                print "Changing location of %s from %s to %s" % (inputFile, se, targetSE)

        #If specified, commit the changes
        if options.change:
            acdcCouch.commitOne(doc)

    return 0

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class DiscreteSummaryHistogramTest(unittest.TestCase):

    def setUp(self):
        """
        _setUp_

        Setup a couch database for testing
        of produced JSON
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("histogram_dump_t")
        self.histogramDB = Database(dbname = "histogram_dump_t")

    def tearDown(self):
        """
        _tearDown_

        Clean the couch
        """
        self.testInit.tearDownCouch()

    def testA_BasicTest(self):
        """
        _testA_BasicTest_

        Build a histogram from a set of discrete data. Check
        that the statistic properties in the histogram are accurate,
        and that this can become a document an uploaded to couch
        """
        # Try and empty one
        histogram = DiscreteSummaryHistogram('SomeTitle', 'Categories')
        histogramJSON = histogram.toJSON()

        self.assertEqual(histogramJSON["title"], "SomeTitle")
        self.assertEqual(histogramJSON["xLabel"], "Categories")
        self.assertFalse(histogramJSON["continuous"])
        self.assertEqual(len(histogramJSON["data"]), 0)
        self.assertEqual(histogramJSON["average"], {})
        self.assertEqual(histogramJSON["stdDev"], {})

        histogram = DiscreteSummaryHistogram('SomeTitle', 'Categories')

        for _ in range(5):
            histogram.addPoint("CategoryA", "FeatureA")
            histogram.addPoint("CategoryB", "FeatureB")

        for _ in range(17):
            histogram.addPoint("CategoryA", "FeatureB")
            histogram.addPoint("CategoryC", "FeatureB")

        for _ in range(3):
            histogram.addPoint("CategoryC", "FeatureA")

        jsonHistogram = histogram.toJSON()

        # Average/stdDev per feature:
        # FeatureA: avg = 2.7 stdev = 2.05
        # FeatureB: avg = 13 stdev = 5.66
        self.assertAlmostEqual(jsonHistogram["average"]["FeatureA"], 2.7, places = 1)
        self.assertAlmostEqual(jsonHistogram["average"]["FeatureB"], 13, places = 1)
        self.assertAlmostEqual(jsonHistogram["stdDev"]["FeatureA"], 2.05, places = 1)
        self.assertAlmostEqual(jsonHistogram["stdDev"]["FeatureB"], 5.66, places = 1)
        self.assertEqual(jsonHistogram["data"]["CategoryA"]["FeatureA"], 5)
        self.assertEqual(jsonHistogram["data"]["CategoryA"]["FeatureB"], 17)
        self.assertEqual(jsonHistogram["data"]["CategoryB"]["FeatureA"], 0)
        self.assertEqual(jsonHistogram["data"]["CategoryB"]["FeatureB"], 5)
        self.assertEqual(jsonHistogram["data"]["CategoryC"]["FeatureA"], 3)
        self.assertEqual(jsonHistogram["data"]["CategoryC"]["FeatureB"], 17)

        # Test couch
        # Try to commit it to couch
        jsonHistogram["_id"] = jsonHistogram["title"]
        self.histogramDB.commitOne(jsonHistogram)

        storedJSON = self.histogramDB.document("SomeTitle")
        self.assertEqual(len(storedJSON["data"]), 3)

        return

# 需要导入模块: from WMCore.Database.CMSCouch import Database [as 别名]
# 或者: from WMCore.Database.CMSCouch.Database import commitOne [as 别名]
class ContinuousSummaryHistogramTest(unittest.TestCase):

    def setUp(self):
        """
        _setUp_

        Setup a couch database for testing
        of produced JSON
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("histogram_dump_t")
        random.seed()
        self.histogramDB = Database(dbname = "histogram_dump_t")

    def tearDown(self):
        """
        _tearDown_

        Clean the couch
        """
        self.testInit.tearDownCouch()

    def buildRandomNumberList(self, n, distribution = "normalvariate", **kwargs):
        """
        _buildRandomNumberList_

        Builds a list with n pseudorandomly distributed
        numbers according to some given distribution
        """
        numberList = []
        if not kwargs:
            kwargs = {"mu" : 0, "sigma" : 1}
        for _ in range(n):
            generator = getattr(random, distribution)
            numberList.append(generator(**kwargs))

        return numberList

    def testA_BasicTest(self):
        """
        _testA_BasicTest_

        Build a histogram from a set of uniformly
        distributed pseudorandom numbers. Check
        that the statistic properties
        in the histogram are accurate to some degree,
        that the histogram binning is done right and
        that this can become a document an uploaded to couch
        """
        inputData = self.buildRandomNumberList(1000)

        histogram = ContinuousSummaryHistogram('TestHisto', 'MyLabel', 'SomeoneElsesLabel')

        # Populate the histogram
        for point in inputData:
            histogram.addPoint(point)

        # Get the JSON
        jsonHistogram = histogram.toJSON()

        # Check the histogram core data
        self.assertEqual(jsonHistogram["title"], "TestHisto")
        self.assertEqual(jsonHistogram["xLabel"], "MyLabel")
        self.assertAlmostEqual(jsonHistogram["average"], 0.0, places = 0)
        self.assertAlmostEqual(jsonHistogram["stdDev"], 1.0, places = 0)
        self.assertEqual(len(jsonHistogram["data"]), 16)
        self.assertTrue(jsonHistogram["continuous"])

        # Check the internal data
        self.assertEqual(jsonHistogram["internalData"]["yLabel"], "SomeoneElsesLabel")
        self.assertEqual(jsonHistogram["internalData"]["nPoints"], 1000)

        # Try to commit it to couch
        jsonHistogram["_id"] = jsonHistogram["title"]
        self.histogramDB.commitOne(jsonHistogram)

        storedJSON = self.histogramDB.document("TestHisto")
        self.assertEqual(len(storedJSON["data"]), 16)

        return

    def testB_extremeData(self):
        """
        _testB_extremeData_

        Put extreme points in the data and try to build a histogram.
        Check that it can process all this correctly
        """

        # First no data
        histogram = ContinuousSummaryHistogram('TestHisto', 'MyLabel', 'SomeoneElsesLabel')
        jsonHistogram = histogram.toJSON()
        self.assertEqual(jsonHistogram["title"], "TestHisto")
        self.assertEqual(jsonHistogram["xLabel"], "MyLabel")
        self.assertEqual(jsonHistogram["average"], 0.0)
        self.assertEqual(jsonHistogram["stdDev"], 0.0)
        self.assertEqual(len(jsonHistogram["data"]), 0)

#.........这里部分代码省略.........