本文整理汇总了Python中WMCore.Database.CMSCouch.Document类的典型用法代码示例。如果您正苦于以下问题:Python Document类的具体用法?Python Document怎么用?Python Document使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Document类的13个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
def __init__(self, workloadId, url, database, workload = None):
Document.__init__(self, workloadId)
self.database = database
self.url = url
self.server = None
self.couchdb = None
if workload != None:
self.update(workload.generateWorkloadSummary())
示例2: __init__
def __init__(self, run, lumi, dataset, user, bfield=0, initial_map = {}):
'''
Instantiate the Couch document and set the appropriate values
'''
CouchDocument.__init__(self, id=self._generate_id(run, lumi, dataset))
self.setdefault('run', run)
self.setdefault('lumi', lumi)
self.setdefault('dataset', dataset.strip('/').split('/'))
self.setdefault('bfield', bfield)
self.setdefault('map_history', [])
self.setdefault('map', initial_map)
self['map']['_meta'] = {'user': user,
'timestamp': str(datetime.now())}
示例3: recordTaskActivity
def recordTaskActivity(self, taskname, comment = ''):
"""Record a task for monitoring"""
try:
record = self.db.document('task_activity')
except CouchNotFoundError:
record = Document('task_activity')
record.setdefault('tasks', {})
record['tasks'].setdefault(taskname, {})
record['tasks'][taskname]['timestamp'] = time.time()
record['tasks'][taskname]['comment'] = comment
try:
self.db.commitOne(record)
except StandardError, ex:
self.logger.error("Unable to update task %s freshness: %s" % (taskname, str(ex)))
示例4: recordTaskActivity
def recordTaskActivity(self, taskname, comment=""):
"""Record a task for monitoring"""
try:
record = self.db.document("task_activity")
except CouchNotFoundError:
record = Document("task_activity")
record.setdefault("tasks", {})
record["tasks"].setdefault(taskname, {})
record["tasks"][taskname]["timestamp"] = time.time()
record["tasks"][taskname]["comment"] = comment
try:
self.db.commitOne(record)
except Exception as ex:
self.logger.error("Unable to update task %s freshness: %s" % (taskname, str(ex)))
示例5: __init__
def __init__(self, couchDB, id = None, elementParams = None):
elementParams = elementParams or {}
WorkQueueElement.__init__(self, **elementParams)
if id:
self._id = id
self._document = Document(id = id)
self._couch = couchDB
示例6: testB
def testB(self):
"""test owning some sample documents"""
u1 = User(name = "evansde77")
g1 = Group(name = "DMWM", administrators = ["evansde77", "drsm79"])
g1.setCouch(self.url, self.database)
g1.connect()
u1.setGroup(g1)
u1.create()
doc1 = Document()
doc1['test-data'] = {"key1" : "value1"}
doc2 = Document()
doc2['test-data'] = {"key2" : "value2"}
id1 = g1.couch.commitOne(doc1)[0]
id2 = g1.couch.commitOne(doc2)[0]
doc1['_id'] = id1[u'id']
doc1['_rev'] = id1[u'rev']
doc2['_id'] = id2[u'id']
doc2['_rev'] = id2[u'rev']
u1.ownThis(doc1)
u1.ownThis(doc2)
self.failUnless(doc1.has_key("owner"))
self.failUnless(doc2.has_key("owner"))
self.failUnless(doc1['owner'].has_key('user'))
self.failUnless(doc1['owner'].has_key('group'))
self.failUnless(doc1['owner']['user'] == u1['name'])
self.failUnless(doc1['owner']['group'] == u1['group'])
self.failUnless(doc2['owner'].has_key('user'))
self.failUnless(doc2['owner'].has_key('group'))
self.failUnless(doc2['owner']['user'] == u1['name'])
self.failUnless(doc2['owner']['group'] == u1['group'])
g1.couch.delete_doc(id1[u'id'])
g1.couch.delete_doc(id2[u'id'])
u1.drop()
g1.drop()
示例7: __init__
def __init__(self, dbURL, couchDBName = None, id = None, rev = None, usePYCurl = False,
ckey = None, cert = None, capath = None, detail = True):
super(ConfigCache, self).__init__()
self.dbname = couchDBName
self.dburl = dbURL
self.detail = detail
try:
self.couchdb = CouchServer(self.dburl, usePYCurl=usePYCurl, ckey=ckey, cert=cert, capath=capath)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname)
except Exception as ex:
msg = "Error connecting to couch: %s\n" % str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
# local cache
self.docs_cache = DocumentCache(self.database, self.detail)
# UserGroup variables
self.group = None
self.owner = None
# Internal data structure
self.document = Document()
self.attachments = {}
self.document['type'] = "config"
self.document['description'] = {}
self.document['description']['config_label'] = None
self.document['description']['config_desc'] = None
if id != None:
self.document['_id'] = id
self.document['pset_tweak_details'] = None
self.document['info'] = None
self.document['config'] = None
return
示例8: CouchWorkQueueElement
class CouchWorkQueueElement(WorkQueueElement):
"""
_CouchWorkQueueElement_
"""
def __init__(self, couchDB, id = None, elementParams = None):
elementParams = elementParams or {}
WorkQueueElement.__init__(self, **elementParams)
if id:
self._id = id
self._document = Document(id = id)
self._couch = couchDB
rev = property(
lambda x: str(x._document[u'_rev']) if x._document.has_key(u'_rev') else x._document.__getitem__('_rev'),
lambda x, newid: x._document.__setitem__('_rev', newid))
timestamp = property(
lambda x: str(x._document[u'timestamp']) if x._document.has_key(u'timestamp') else x._document.__getitem__('timestamp')
)
updatetime = property(
lambda x: str(x._document[u'updatetime']) if x._document.has_key(u'updatetime') else 0
)
@classmethod
def fromDocument(cls, couchDB, doc):
"""Create element from couch document"""
element = CouchWorkQueueElement(couchDB = couchDB,
id = doc['_id'],
elementParams = doc.pop('WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement')
)
element._document['_rev'] = doc.pop('_rev')
element._document['timestamp'] = doc.pop('timestamp')
element._document['updatetime'] = doc.pop('updatetime')
return element
def save(self):
"""
_save
"""
self.populateDocument()
self._couch.queue(self._document)
def load(self):
"""
_load_
Load the document representing this WQE
"""
document = self._couch.document(self._document['_id'])
self.update(document.pop('WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'))
self._document['_rev'] = document.pop('_rev')
self._document['timestamp'] = document.pop('timestamp', None)
self._document['updatetime'] = document.pop('updatetime', None)
return self
def delete(self):
"""Delete element"""
self.populateDocument()
self._document.delete()
self._couch.queue(self._document)
def populateDocument(self):
"""Certain attributed shouldn't be stored"""
self._document.update(self.__to_json__(None))
now = time.time()
self._document['updatetime'] = now
self._document.setdefault('timestamp', now)
if not self._document.get('_id') and self.id:
self._document['_id'] = self.id
attrs = ['WMSpec', 'Task']
for attr in attrs:
self._document['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'].pop(attr, None)
示例9: ConfigCache
class ConfigCache(WMObject):
"""
_ConfigCache_
The class that handles the upload and download of configCache
artifacts from Couch
"""
def __init__(self, dbURL, couchDBName = None, id = None, rev = None, usePYCurl = False,
ckey = None, cert = None, capath = None, detail = True):
self.dbname = couchDBName
self.dburl = dbURL
self.detail = detail
try:
self.couchdb = CouchServer(self.dburl, usePYCurl=usePYCurl, ckey=ckey, cert=cert, capath=capath)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname)
except Exception as ex:
msg = "Error connecting to couch: %s\n" % str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
# local cache
self.docs_cache = DocumentCache(self.database, self.detail)
# UserGroup variables
self.group = None
self.owner = None
# Internal data structure
self.document = Document()
self.attachments = {}
self.document['type'] = "config"
self.document['description'] = {}
self.document['description']['config_label'] = None
self.document['description']['config_desc'] = None
if id != None:
self.document['_id'] = id
self.document['pset_tweak_details'] = None
self.document['info'] = None
self.document['config'] = None
return
def createDatabase(self):
"""
_createDatabase_
"""
database = self.couchdb.createDatabase(self.dbname)
database.commit()
return database
def connectUserGroup(self, groupname, username):
"""
_connectUserGroup_
"""
self.group = Group(name = groupname)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.owner = makeUser(groupname, username,
couchUrl = self.dburl,
couchDatabase = self.dbname)
return
def createUserGroup(self, groupname, username):
"""
_createUserGroup_
Create all the userGroup information
"""
self.createGroup(name = groupname)
self.createUser(username = username)
return
def createGroup(self, name):
"""
_createGroup_
Create Group for GroupUser
"""
self.group = Group(name = name)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.group.create()
return
def setLabel(self, label):
"""
_setLabel_
Util to add a descriptive label to the configuration doc
"""
self.document['description']['config_label'] = label
def setDescription(self, desc):
"""
#.........这里部分代码省略.........
示例10: __init__
def __init__(self, dqis_db=None, savable=False, *args, **kwargs):
Document.__init__(self, *args, **kwargs)
self.dqis_db = dqis_db
示例11: gatherWMDataMiningStats
def gatherWMDataMiningStats(wmstatsUrl, reqmgrUrl, wmminigUrl, archived = False, log = logging.info):
server, database = splitCouchServiceURL(wmminigUrl)
analyticsServer = CouchServer(server)
couchdb = analyticsServer.connectDatabase(database)
WMStats = WMStatsReader(wmstatsUrl)
reqMgrServer, reqMgrDB = splitCouchServiceURL(reqmgrUrl)
reqMgr = CouchServer(reqMgrServer).connectDatabase(reqMgrDB, False)
if archived:
funcName = "Archived Requests"
else:
funcName = "Active Requests"
log("INFO: %s: Getting job information from %s and %s. Please wait." % (
funcName, wmstatsUrl, reqmgrUrl))
if archived:
checkStates = ['normal-archived', 'rejected-archived', 'aborted-archived']
jobInfoFlag = False
else:
checkStates = WMStatsReader.ACTIVE_STATUS
jobInfoFlag = True
requests = WMStats.getRequestByStatus(checkStates, jobInfoFlag = jobInfoFlag)
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
requestsDict = requestCollection.getData()
log("INFO: %s: Total %s requests retrieved\n" % (funcName, len(result)))
report = {}
for wf in result.keys():
# Store a copy of the CouchDB document so we can compare later before updating
if couchdb.documentExists(wf):
oldCouchDoc = couchdb.document(wf)
wfExists = True
else:
oldCouchDoc = CouchDoc(id=wf)
wfExists = False
newCouchDoc = copy.deepcopy(oldCouchDoc)
ancientCouchDoc = copy.deepcopy(oldCouchDoc)
report[wf] = oldCouchDoc
# FIXME: remove report, only have two instances of couchDoc
if not oldCouchDoc.has_key('filterEfficiency') or not oldCouchDoc.has_key('runWhiteList'):
runWhiteList = []
filterEfficiency = None
try:
#log("DEBUG: Looking up %s in ReqMgr" % wf)
rmDoc = reqMgr.document(wf)
runWhiteList = rmDoc.get('RunWhiteList', [])
filterEfficiency = rmDoc.get('FilterEfficiency', None)
except:
pass # ReqMgr no longer has the workflow
report[wf].update({'filterEfficiency':filterEfficiency, 'runWhiteList':runWhiteList})
# Basic parameters of the workflow
priority = requests[wf]['priority']
requestType = requests[wf]['request_type']
targetLumis = requests[wf].get('input_lumis', 0)
targetEvents = requests[wf].get('input_events', 0)
campaign = requests[wf]['campaign']
prep_id = requests[wf].get('prep_id', None)
outputdatasets = requests[wf].get('outputdatasets', [])
# Can be an empty list, full list, empty string, or non-empty string!
inputdataset = requests[wf]['inputdataset']
if isinstance(inputdataset, (list,)):
if inputdataset:
inputdataset = inputdataset[0]
else:
inputdataset = ''
outputTier = 'Unknown'
try:
outputTiers = []
for ds in outputdatasets:
if type(ds) == list:
outputTiers.append(ds[0].split('/')[-1])
else:
outputTiers.append(ds.split('/')[-1])
except:
log("ERROR: Could not decode outputdatasets: %s" % outputdatasets) # Sometimes is a list of lists, not just a list. Bail
if inputdataset:
inputTier = inputdataset.split('/')[-1]
if inputTier in ['GEN']:
outputTier = 'LHE'
elif inputTier in ['RAW', 'RECO']:
outputTier = 'AOD'
elif inputTier in ['GEN-SIM']:
outputTier = 'AODSIM'
elif 'AODSIM' in outputTiers:
outputTier = 'AODSIM'
else:
#.........这里部分代码省略.........
示例12: gatherWMDataMiningStats
def gatherWMDataMiningStats(wmstatsUrl, reqmgrUrl, wmMiningUrl,
mcmUrl, mcmCert, mcmKey, tmpDir,
archived = False, log = logging.info):
server, database = splitCouchServiceURL(wmMiningUrl)
analyticsServer = CouchServer(server)
couchdb = analyticsServer.connectDatabase(database)
WMStats = WMStatsReader(wmstatsUrl, reqmgrUrl, reqdbCouchApp = "ReqMgr")
reqMgrServer, reqMgrDB = splitCouchServiceURL(reqmgrUrl)
reqMgr = CouchServer(reqMgrServer).connectDatabase(reqMgrDB, False)
if archived:
funcName = "Archived Requests"
else:
funcName = "Active Requests"
log.info("%s: Getting job information from %s and %s. Please wait." % (
funcName, wmstatsUrl, reqmgrUrl))
if archived:
checkStates = ['normal-archived', 'rejected-archived', 'aborted-archived']
jobInfoFlag = False
else:
checkStates = WMStatsReader.ACTIVE_STATUS
jobInfoFlag = True
requests = WMStats.getRequestByStatus(checkStates, jobInfoFlag = jobInfoFlag, legacyFormat = True)
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
requestsDict = requestCollection.getData()
log.info("%s: Total %s requests retrieved\n" % (funcName, len(result)))
report = {}
nMCMCalls = 0
with McM(cert=mcmCert, key=mcmKey, url=mcmUrl, tmpDir=tmpDir) as mcm:
for wf in result.keys():
# Store a copy of the CouchDB document so we can compare later before updating
if couchdb.documentExists(wf):
oldCouchDoc = couchdb.document(wf)
wfExists = True
else:
oldCouchDoc = CouchDoc(id=wf)
wfExists = False
newCouchDoc = copy.deepcopy(oldCouchDoc)
ancientCouchDoc = copy.deepcopy(oldCouchDoc)
report[wf] = oldCouchDoc
# FIXME: remove report, only have two instances of couchDoc
if 'filterEfficiency' not in oldCouchDoc or 'runWhiteList' not in oldCouchDoc:
runWhiteList = []
filterEfficiency = None
try:
#log.debug("Looking up %s in ReqMgr" % wf)
rmDoc = reqMgr.document(wf)
runWhiteList = rmDoc.get('RunWhiteList', [])
filterEfficiency = rmDoc.get('FilterEfficiency', None)
except:
pass # ReqMgr no longer has the workflow
report[wf].update({'filterEfficiency':filterEfficiency, 'runWhiteList':runWhiteList})
if ('mcmTotalEvents' not in oldCouchDoc or
'mcmApprovalTime' not in oldCouchDoc or
oldCouchDoc.get('mcmTotalEvents', 'Unknown') == 'Unknown' or
oldCouchDoc.get('mcmApprovalTime', 'Unknown') == 'Unknown'):
prepID = oldCouchDoc.get('prepID', None)
if prepID and nMCMCalls <= maxMCMCalls:
log.info("Trying to update McM info for %s, PREPID %s" % (wf, prepID))
# Get information from McM. Don't call too many times, can take a long time
nMCMCalls += 1
try:
mcmHistory = mcm.getHistory(prepID = prepID)
if 'mcmApprovalTime' not in oldCouchDoc:
report[wf].update({'mcmApprovalTime':'NoMcMData'})
found = False
for entry in mcmHistory:
if entry['action'] == 'set status' and entry['step'] == 'announced':
dateString = entry['updater']['submission_date']
dt = datetime.strptime(dateString, '%Y-%m-%d-%H-%M')
report[wf].update({'mcmApprovalTime':time.mktime(dt.timetuple())})
found = True
if not found:
log.error("History found but no approval time for %s" % wf)
except McMNoDataError:
log.error("Setting NoMcMData for %s" % wf)
report[wf].update({'mcmApprovalTime':'NoMcMData'})
except (RuntimeError, IOError):
exc_type, exc_value, exc_traceback = sys.exc_info()
log.error("%s getting history from McM for PREP ID %s. May be transient and/or SSO problem." %
(exc_type, prepID))
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
log.error("%s getting history from McM for PREP ID %s. Unknown error." %
(exc_type, prepID))
#.........这里部分代码省略.........
示例13: RequestInfoCollection
requests = WMStats.getRequestByStatus(checkStates, jobInfoFlag = jobInfoFlag)
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
requestsDict = requestCollection.getData()
print "Total %s requests retrieved\n" % len(result)
report = {}
for wf in result.keys():
# Store a copy of the CouchDB document so we can compare later before updating
if couchdb.documentExists(wf):
oldCouchDoc = couchdb.document(wf)
wfExists = True
else:
oldCouchDoc = CouchDoc(id=wf)
wfExists = False
newCouchDoc = copy.deepcopy(oldCouchDoc)
ancientCouchDoc = copy.deepcopy(oldCouchDoc)
report[wf] = oldCouchDoc
# FIXME: remove report, only have two instances of couchDoc
if not oldCouchDoc.has_key('filterEfficiency') or not oldCouchDoc.has_key('runWhiteList'):
runWhiteList = []
filterEfficiency = None
try:
print "Looking up %s in ReqMgr" % wf
rmDoc = reqMgr.document(wf)
runWhiteList = rmDoc.get('RunWhiteList', [])
filterEfficiency = rmDoc.get('FilterEfficiency', None)