Python LogDB.LogDB类代码示例

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.
        """
        logging.basicConfig()
        self.logger = logging.getLogger('LogDBTest')
#        self.logger.setLevel(logging.DEBUG)

        self.schema = []
        self.couchApps = ["LogDB"]
        self.testInit = TestInitCouchApp('LogDBTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'logdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        url = "%s/%s" % (self.testInit.couchUrl, dbName)

        identifier = 'agentname'
        self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test")
        identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
        self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.report = LogDBReport(self.agent_inst)

 def logDBCleanUp(self, config):
     """
     gather active data statistics
     """
     
     logdb = LogDB(config.central_logdb_url, config.log_reporter)
     logdb.cleanup(config.logDBCleanDuration)
     
     cherrypy.log("cleaned up log db")        
     return

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection. 
        """
        logging.basicConfig()
        self.logger = logging.getLogger('LogDBTest')
#        self.logger.setLevel(logging.DEBUG)
        url = 'http://localhost:5984/logdb_t'
        identifier = 'agentname'
        self.agent_inst = LogDB(url, identifier, logger=self.logger, create=True)
        self.agent_inst2 = LogDB(url, identifier, logger=self.logger, create=True, thread_name="Test")
        identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
        self.user_inst = LogDB(url, identifier, logger=self.logger, create=True)
        self.report = LogDBReport(self.agent_inst)

class LogDBTest(unittest.TestCase):
    """
    _LogDBTest_

    """
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection. 
        """
        logging.basicConfig()
        self.logger = logging.getLogger('LogDBTest')
#        self.logger.setLevel(logging.DEBUG)
        url = 'http://localhost:5984/logdb_t'
        identifier = 'agentname'
        self.agent_inst = LogDB(url, identifier, logger=self.logger, create=True)
        self.agent_inst2 = LogDB(url, identifier, logger=self.logger, create=True, thread_name="Test")
        identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
        self.user_inst = LogDB(url, identifier, logger=self.logger, create=True)
        self.report = LogDBReport(self.agent_inst)

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.agent_inst.backend.deleteDatabase()

    def test_docid(self):
        "Test docid API"
        request = 'abs'
        mtype = self.agent_inst.backend.prefix('info')
        res = self.agent_inst.backend.docid(request, mtype)
        key = '--'.join([request, self.agent_inst.identifier, self.agent_inst.thread_name, mtype])
        khash = gen_hash(key)
        self.assertEqual(res, khash)

    def test_prefix(self):
        "Test prefix LogDBBackend API"
        request = 'abc'

        self.assertEqual(self.agent_inst.agent, 1)
        mtype = self.agent_inst.backend.prefix('msg')
        self.assertEqual(mtype, 'agent-msg')

        self.assertEqual(self.user_inst.agent, 0)
        mtype = self.user_inst.backend.prefix('msg')
        self.assertEqual(mtype, 'msg')

    def test_apis(self):
        "Test LogDB APIs"
        request = 'abc'

        # if we post messages for the same request only last one should survive
        self.agent_inst.post(request, 'msg1')
#        time.sleep(1)
        self.agent_inst.post(request, 'msg2')
#        time.sleep(1)
        self.agent_inst.post(request, 'msg3')
#        time.sleep(1)
        self.agent_inst.post(request, 'msg4')
#        time.sleep(1)
        docs = self.agent_inst.get(request)
        self.assertEqual(len(docs), 1)
        self.assertEqual(docs[0]['msg'], 'msg4')

        # add message as user, user based messages will grow
        self.user_inst.post(request, 'doc1')
#        time.sleep(1)
        docs = self.user_inst.get(request)
        self.assertEqual(len(docs), 2) # we have msg4 and doc1
        self.user_inst.post(request, 'doc1') # we have msg2 and two doc1 messages
#        time.sleep(1)
        docs = self.user_inst.get(request)
        self.assertEqual(len(docs), 3)

    def test_cleanup(self):
        "Test clean-up LogDB API"
        request = 'abc'
        self.agent_inst.post(request, 'msg1', 'info')
        self.agent_inst.post(request, 'msg2', 'comment')
        self.agent_inst.post(request, 'msg3', 'warning')
        self.agent_inst.post(request, 'msg4', 'error')
        all_docs = self.agent_inst.get(request)
        self.agent_inst.backend.cleanup(thr=10) # look into past
        past_docs = self.agent_inst.get(request)
        self.assertEqual(len(all_docs), len(past_docs))
        self.agent_inst.backend.cleanup(thr=-10) # look into future
        docs = self.agent_inst.get(request)
        self.assertEqual(len(docs), 0)

    def test_get_all_requests(self):
        "Test get_all_requests LogDB API"
        request = 'abc'
        self.agent_inst.post(request, 'msg1', 'info')
        self.agent_inst.post(request, 'msg2', 'info')
        self.agent_inst.post(request, 'msg3', 'warning')
        self.agent_inst.post(request, 'msg4', 'error')
#.........这里部分代码省略.........

    def __init__(self, app, config, mount):
        self.base = config.base
        self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
        if config and not isinstance(config, dict):
            web_config = config.dictionary_()
        if not config:
            web_config = {'base': self.base}
        TemplatedPage.__init__(self, web_config)
        imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
        self.imgdir = web_config.get('imgdir', imgdir)
        cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
        self.cssdir = web_config.get('cssdir', cssdir)
        jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
        self.jsdir = web_config.get('jsdir', jsdir)
        spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
        self.spdir = web_config.get('spdir', spdir)
        # read scripts area and initialize data-ops scripts
        self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
        self.sdir = web_config.get('sdir', self.sdir)
        self.sdict_thr = web_config.get('sdict_thr', 600)  # put reasonable 10 min interval
        self.sdict = {'ts': time.time()}  # placeholder for data-ops scripts
        self.update_scripts(force=True)

        # To be filled at run time
        self.cssmap = {}
        self.jsmap = {}
        self.imgmap = {}
        self.yuimap = {}

        std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
        self.std_specs = spec_list(std_specs_dir)
        self.std_specs.sort()

        # Update CherryPy configuration
        mime_types = ['text/css']
        mime_types += ['application/javascript', 'text/javascript',
                       'application/x-javascript', 'text/x-javascript']
        cherryconf.update({'tools.encode.on': True,
                           'tools.gzip.on': True,
                           'tools.gzip.mime_types': mime_types,
                           })
        self._cache = {}

        # initialize access to reqmgr2 APIs
        self.reqmgr_url = config.reqmgr.reqmgr2_url
        self.reqmgr = ReqMgr(self.reqmgr_url)
        # only gets current view (This might cause to reponse time much longer,
        # If upto date view is not needed overwrite Fale)
        self.reqmgr._noStale = True

        # get fields which we'll use in templates
        cdict = config.reqmgr.dictionary_()
        self.couch_url = cdict.get('couch_host', '')
        self.couch_dbname = cdict.get('couch_reqmgr_db', '')
        self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
        self.acdc_url = cdict.get('acdc_host', '')
        self.acdc_dbname = cdict.get('acdc_db', '')
        self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
        self.dbs_url = cdict.get('dbs_url', '')
        self.dqm_url = cdict.get('dqm_url', '')
        self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
        self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')

        # LogDB holder
        centralurl = cdict.get("central_logdb_url", "")
        identifier = cdict.get("log_reporter", "reqmgr2")
        self.logdb = LogDB(centralurl, identifier)

        # local team cache which will request data from wmstats
        base, uri = self.reqmgr_url.split('://')
        base_url = '%s://%s' % (base, uri.split('/')[0])
        self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url)
        if not self.wmstatsurl:
            raise Exception('ReqMgr2 configuration file does not provide wmstats url')
        self.team_cache = []

        # fetch assignment arguments specification from StdBase
        self.assignArgs = StdBase().getWorkloadAssignArgs()
        self.assignArgs = {key: val['default'] for key, val in self.assignArgs.items()}

class ReqMgrService(TemplatedPage):
    """
    Request Manager web service class
    """

    def __init__(self, app, config, mount):
        self.base = config.base
        self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
        if config and not isinstance(config, dict):
            web_config = config.dictionary_()
        if not config:
            web_config = {'base': self.base}
        TemplatedPage.__init__(self, web_config)
        imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
        self.imgdir = web_config.get('imgdir', imgdir)
        cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
        self.cssdir = web_config.get('cssdir', cssdir)
        jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
        self.jsdir = web_config.get('jsdir', jsdir)
        spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
        self.spdir = web_config.get('spdir', spdir)
        # read scripts area and initialize data-ops scripts
        self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
        self.sdir = web_config.get('sdir', self.sdir)
        self.sdict_thr = web_config.get('sdict_thr', 600)  # put reasonable 10 min interval
        self.sdict = {'ts': time.time()}  # placeholder for data-ops scripts
        self.update_scripts(force=True)

        # To be filled at run time
        self.cssmap = {}
        self.jsmap = {}
        self.imgmap = {}
        self.yuimap = {}

        std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
        self.std_specs = spec_list(std_specs_dir)
        self.std_specs.sort()

        # Update CherryPy configuration
        mime_types = ['text/css']
        mime_types += ['application/javascript', 'text/javascript',
                       'application/x-javascript', 'text/x-javascript']
        cherryconf.update({'tools.encode.on': True,
                           'tools.gzip.on': True,
                           'tools.gzip.mime_types': mime_types,
                           })
        self._cache = {}

        # initialize access to reqmgr2 APIs
        self.reqmgr_url = config.reqmgr.reqmgr2_url
        self.reqmgr = ReqMgr(self.reqmgr_url)
        # only gets current view (This might cause to reponse time much longer,
        # If upto date view is not needed overwrite Fale)
        self.reqmgr._noStale = True

        # get fields which we'll use in templates
        cdict = config.reqmgr.dictionary_()
        self.couch_url = cdict.get('couch_host', '')
        self.couch_dbname = cdict.get('couch_reqmgr_db', '')
        self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
        self.acdc_url = cdict.get('acdc_host', '')
        self.acdc_dbname = cdict.get('acdc_db', '')
        self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
        self.dbs_url = cdict.get('dbs_url', '')
        self.dqm_url = cdict.get('dqm_url', '')
        self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
        self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')

        # LogDB holder
        centralurl = cdict.get("central_logdb_url", "")
        identifier = cdict.get("log_reporter", "reqmgr2")
        self.logdb = LogDB(centralurl, identifier)

        # local team cache which will request data from wmstats
        base, uri = self.reqmgr_url.split('://')
        base_url = '%s://%s' % (base, uri.split('/')[0])
        self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url)
        if not self.wmstatsurl:
            raise Exception('ReqMgr2 configuration file does not provide wmstats url')
        self.team_cache = []

        # fetch assignment arguments specification from StdBase
        self.assignArgs = StdBase().getWorkloadAssignArgs()
        self.assignArgs = {key: val['default'] for key, val in self.assignArgs.items()}

    def getTeams(self):
        "Helper function to get teams from wmstats or local cache"
        teams = self.team_cache
        url = '%s/data/teams' % self.wmstatsurl
        params = {}
        headers = {'Accept': 'application/json'}
        try:
            data = getdata(url, params, headers)
            if 'error' in data:
                print("WARNING: fail to get teams from %s" % url)
                print(data)
            teams = data.get('result', [])
            self.team_cache = teams
        except Exception as exp:
            print("WARNING: fail to get teams from %s" % url)
#.........这里部分代码省略.........

    def __init__(self, app, config, mount):
        self.base = config.base
        self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
        if config and not isinstance(config, dict):
            web_config = config.dictionary_()
        if not config:
            web_config = {'base': self.base}
        TemplatedPage.__init__(self, web_config)
        imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
        self.imgdir = web_config.get('imgdir', imgdir)
        cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
        self.cssdir = web_config.get('cssdir', cssdir)
        jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
        self.jsdir = web_config.get('jsdir', jsdir)
        spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
        self.spdir = web_config.get('spdir', spdir)
        # read scripts area and initialize data-ops scripts
        self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
        self.sdir = web_config.get('sdir', self.sdir)
        self.sdict_thr = web_config.get('sdict_thr', 600)  # put reasonable 10 min interval
        self.sdict = {'ts': time.time()}  # placeholder for data-ops scripts
        self.update_scripts(force=True)

        # To be filled at run time
        self.cssmap = {}
        self.jsmap = {}
        self.imgmap = {}
        self.yuimap = {}

        std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
        self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
        self.std_specs.sort()

        # Update CherryPy configuration
        mime_types = ['text/css']
        mime_types += ['application/javascript', 'text/javascript',
                       'application/x-javascript', 'text/x-javascript']
        cherryconf.update({'tools.encode.on': True,
                           'tools.gzip.on': True,
                           'tools.gzip.mime_types': mime_types,
                           })
        self._cache = {}

        # initialize rest API
        statedir = '/tmp'
        app = RESTMain(config, statedir)  # REST application
        mount = '/rest'  # mount point for cherrypy service
        api = RestApiHub(app, config.reqmgr, mount)

        # initialize access to reqmgr2 APIs
        self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
        # only gets current view (This might cause to reponse time much longer, 
        # If upto date view is not needed overwrite Fale)
        self.reqmgr._noStale = True

        # admin helpers
        self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info')
        self.admin_group = Group(app, api, config.reqmgr, mount=mount + '/group')
        self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team')

        # get fields which we'll use in templates
        cdict = config.reqmgr.dictionary_()
        self.couch_url = cdict.get('couch_host', '')
        self.couch_dbname = cdict.get('couch_reqmgr_db', '')
        self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
        self.acdc_url = cdict.get('acdc_host', '')
        self.acdc_dbname = cdict.get('acdc_db', '')
        self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
        self.dbs_url = cdict.get('dbs_url', '')
        self.dqm_url = cdict.get('dqm_url', '')
        self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
        self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')

        # LogDB holder
        centralurl = cdict.get("central_logdb_url", "")
        identifier = cdict.get("log_reporter", "reqmgr2")
        self.logdb = LogDB(centralurl, identifier)

class ReqMgrService(TemplatedPage):
    """
    Request Manager web service class
    """

    def __init__(self, app, config, mount):
        self.base = config.base
        self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
        if config and not isinstance(config, dict):
            web_config = config.dictionary_()
        if not config:
            web_config = {'base': self.base}
        TemplatedPage.__init__(self, web_config)
        imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
        self.imgdir = web_config.get('imgdir', imgdir)
        cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
        self.cssdir = web_config.get('cssdir', cssdir)
        jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
        self.jsdir = web_config.get('jsdir', jsdir)
        spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
        self.spdir = web_config.get('spdir', spdir)
        # read scripts area and initialize data-ops scripts
        self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
        self.sdir = web_config.get('sdir', self.sdir)
        self.sdict_thr = web_config.get('sdict_thr', 600)  # put reasonable 10 min interval
        self.sdict = {'ts': time.time()}  # placeholder for data-ops scripts
        self.update_scripts(force=True)

        # To be filled at run time
        self.cssmap = {}
        self.jsmap = {}
        self.imgmap = {}
        self.yuimap = {}

        std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
        self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
        self.std_specs.sort()

        # Update CherryPy configuration
        mime_types = ['text/css']
        mime_types += ['application/javascript', 'text/javascript',
                       'application/x-javascript', 'text/x-javascript']
        cherryconf.update({'tools.encode.on': True,
                           'tools.gzip.on': True,
                           'tools.gzip.mime_types': mime_types,
                           })
        self._cache = {}

        # initialize rest API
        statedir = '/tmp'
        app = RESTMain(config, statedir)  # REST application
        mount = '/rest'  # mount point for cherrypy service
        api = RestApiHub(app, config.reqmgr, mount)

        # initialize access to reqmgr2 APIs
        self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
        # only gets current view (This might cause to reponse time much longer, 
        # If upto date view is not needed overwrite Fale)
        self.reqmgr._noStale = True

        # admin helpers
        self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info')
        self.admin_group = Group(app, api, config.reqmgr, mount=mount + '/group')
        self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team')

        # get fields which we'll use in templates
        cdict = config.reqmgr.dictionary_()
        self.couch_url = cdict.get('couch_host', '')
        self.couch_dbname = cdict.get('couch_reqmgr_db', '')
        self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
        self.acdc_url = cdict.get('acdc_host', '')
        self.acdc_dbname = cdict.get('acdc_db', '')
        self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
        self.dbs_url = cdict.get('dbs_url', '')
        self.dqm_url = cdict.get('dqm_url', '')
        self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
        self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')

        # LogDB holder
        centralurl = cdict.get("central_logdb_url", "")
        identifier = cdict.get("log_reporter", "reqmgr2")
        self.logdb = LogDB(centralurl, identifier)

    def user(self):
        """
        Return user name associated with this instance.
        """
        try:
            return cherrypy.request.user['login']
        except:
            return 'testuser'

    def user_dn(self):
        "Return user DN"
        try:
            return cherrypy.request.user['dn']
        except:
            return '/CN/bla/foo'

    def update_scripts(self, force=False):
#.........这里部分代码省略.........