Python PrintManager.info方法代码示例

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
 def test_info(self):
     "Test logger info method"
     old_stdout = sys.stdout
     logger = PrintManager(self.name, verbose=1)
     sys.stdout = StringIO()
     logger.info('test')
     result = sys.stdout.getvalue()
     expect = 'INFO %s:%s test\n' % (self.name, funcname())
     self.assertEqual(expect, result)
     sys.stdout = old_stdout

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class QueryRunner(object):
    "Replaces das_robot"
    task_options = [{'name':'query', 'type':'string', 'default':None,
                   'help':'Query to issue using das_core::call'}]
    def __init__(self, **kwargs):
        self.logger = PrintManager('QueryRunner', kwargs.get('verbose', 0))
        self.das = kwargs['DAS']
        self.dasquery = DASQuery(kwargs['dasquery'])
    def __call__(self):
        "__call__ implementation"
        self.logger.info("Issuing query %s" % self.dasquery)
        result = self.das.call(self.dasquery, add_to_analytics=False)
        return {'result':result}

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class Test(object):
    """
    This is a test task that prints a message (supplied in kwargs) to stdout.
    It will randomly sometimes raise exceptions (5%), spawn subtasks (4%) or
    disable resubmission of itself (1%).
    """
    task_title = "Test task"
    task_options = [{"name":"message", "type":"string",
                     "default":"hello world", "help":"Message to print"}]

    def __init__(self, **kwargs):
        self.logger = PrintManager('Test', kwargs.get('verbose', 0))
        self.name = kwargs['name']
        self.message = kwargs['message']
        self.index = kwargs['index']

    def __call__(self):
        self.logger.info('%s from index=%s' % (self.message, self.index))

        result = {}

        effect = random.random()
        if effect > 0.99:
            self.logger.info('..disabling resubmission')
            result['resubmit'] = False
        elif effect > 0.95:
            task = {'name':'spawn-of-%s' % self.index,
                    'classname':'Test',
                    'interval': random.randint(1,30),
                    'kwargs':{'message':'spawn-of-%s' % self.message}}
            effect2 = random.random()
            if effect2 > 0.50:
                task['only_once'] =  True
                self.logger.info('..spawning run-once task')
            elif effect2 > 0.25:
                task['max_runs'] = random.randint(1, 5)
                self.logger.info(\
                '..spawning task to run %s times' % task['max_runs'])
            else:
                task['only_before'] = time.time() + random.randint(1, 120)
                self.logger.info(\
                '..spawning task to run until %s' % task['only_before'])
            result['new_tasks'] = [task]
        elif effect > 0.90:
            self.logger.error('..raising an exception')
            raise

        return result

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASKeyLearning(object):
    """
    This class manages DAS key-learning DB.
    
    Key-learning is an intermittent process (triggered infrequently
    by a task running in the analytics framework), which involves
    searching through the raw cache for (a subset of but with
    maximum primary key coverage) all output documents, generating
    the set of all data members (in a dotted-dict fashion) and storing
    those as primary-key:data-member records (with an associated
    last-updated-time).
    
    """
    def __init__(self, config):
        self.verbose  = config['verbose']
        self.logger   = PrintManager('DASKeyLearning', self.verbose)
        self.services = config['services']
        self.dburi    = config['mongodb']['dburi']
        self.dbname   = config['keylearningdb']['dbname']
        self.colname  = config['keylearningdb']['collname']
        
        self.mapping  = config['dasmapping']

        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)
        
        self.col = None
        self.create_db()
        
        

    def create_db(self):
        """
        Establish connection to MongoDB back-end and create DB.
        """
        self.col = db_connection(self.dburi)[self.dbname][self.colname]
        
    def add_members(self, system, urn, members):
        """
        Add a list of data members for a given API (system, urn, url),
        and generate, which are stored as separate records.
        """
        msg = "system=%s, urn=%s, members=%s)" % (system, urn, members)
        self.logger.info(msg)
        
        result = self.col.find_one({'system': system, 'urn': urn})
        if result:       
            self.col.update({'_id': result['_id']},
                            {'$addToSet': {'members': {'$each': members}}})
        else:
            keys = self.mapping.api2daskey(system, urn)
            self.col.insert({'system': system,
                             'urn': urn,
                             'keys': keys,
                             'members': members})
                
        for member in members:
            if not self.col.find_one({'member': member}):
                self.col.insert({'member': member,
                                 'stems': self.stem(member)})
                
        index_list = [('system', 1), ('urn', 1), ('members', 1), ('stems', 1)]
        create_indexes(self.col, index_list)
        
    def stem(self, member):
        """
        Produce an extended set of strings which can be used for text-search.
        TODO: Use PyStemmer or something more sophisticated here.
        """
        
        return member.lower().split('.')
    
    def text_search(self, text):
        """
        Perform a text search for data members matching a string. The input is
        split if it already includes dotted elements (in which case we need to find
        a member matching all the split elements), otherwise we look for any member
        whose stem list contains the text.
        """
        text = text.lower()
        if '.' in text:
            possible_members = self.col.find({'stems': {'$all': text.split('.')}}, 
                                             fields=['member'])
        else:
            possible_members = self.col.find({'stems': text}, 
                                             fields=['member'])
        
        return [doc['member'] for doc in possible_members]
        
    
    def member_info(self, member):
        """
        Once the text search has identified a member that might be a match,
        return which systems, APIs and hence DAS keys this points to.
        """
        result = []
        for doc in self.col.find({'members': member}, 
                                 fields=['system', 'urn', 'keys']):
            
            result.append({'system': doc['system'],
#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASMongocache(object):
    """
    DAS cache based MongoDB. 
    """
    def __init__(self, config):
        self.emptyset_expire = expire_timestamp(\
            config['das'].get('emptyset_expire', 5))
        self.dburi   = config['mongodb']['dburi']
        self.cache_size = config['mongodb']['bulkupdate_size']
        self.dbname  = config['dasdb']['dbname']
        self.verbose = config['verbose']
        self.logger  = PrintManager('DASMongocache', self.verbose)
        self.mapping = config['dasmapping']

        self.conn    = db_connection(self.dburi)
        self.mdb     = self.conn[self.dbname]
        self.col     = self.mdb[config['dasdb']['cachecollection']]
        self.mrcol   = self.mdb[config['dasdb']['mrcollection']]
        self.merge   = self.mdb[config['dasdb']['mergecollection']]
        self.gfs     = db_gridfs(self.dburi)

        self.logdb   = DASLogdb(config)

        self.das_internal_keys = ['das_id', 'das', 'cache_id', 'qhash']

        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)

        self.add_manipulator()

        # ensure that we have the following indexes
        index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
                      ('das.system', ASCENDING),
                      ('qhash', DESCENDING),
                      ('das.empty_record', ASCENDING)]
        create_indexes(self.col, index_list)
        index_list = [('das.expire', ASCENDING), ('das_id', ASCENDING),
                      ('qhash', DESCENDING),
                      ('das.empty_record', ASCENDING), ('das.ts', ASCENDING)]
        create_indexes(self.merge, index_list)
        
    def add_manipulator(self):
        """
        Add DAS-specific MongoDB SON manipulator to perform
        conversion of inserted data into DAS cache.
        """
        das_son_manipulator = DAS_SONManipulator()
        self.mdb.add_son_manipulator(das_son_manipulator)
        msg = "DAS_SONManipulator %s" \
        % das_son_manipulator
        self.logger.debug(msg)

    def similar_queries(self, dasquery):
        """
        Check if we have query results in cache whose conditions are
        superset of provided query. The method only works for single
        key whose value is substring of value in input query.
        For example, if cache contains records about T1 sites, 
        then input query T1_CH_CERN is subset of results stored in cache.
        """
        spec = dasquery.mongo_query.get('spec', {})
        cond = {'query.spec.key': {'$in' : spec.keys()}, 'qhash':dasquery.qhash}
        for row in self.col.find(cond):
            found_query = DASQuery(row['query'])
            if  dasquery.qhash == found_query.qhash:
                msg = "%s similar to %s" % (dasquery, found_query)
                self.logger.info(msg)
                return found_query
        return False
    
    def get_superset_keys(self, key, value):
        """
        This is a special-case version of similar_keys,
        intended for analysers that want to quickly
        find possible superset queries of a simple
        query of the form key=value.
        """
        
        msg = "%s=%s" % (key, value)
        self.logger.debug(msg)
        cond = {'query.spec.key': key}
        for row in self.col.find(cond):
            mongo_query = decode_mongo_query(row['query'])
            for thiskey, thisvalue in mongo_query.iteritems():
                if thiskey == key:
                    if fnmatch.fnmatch(value, thisvalue):
                        yield thisvalue

    def get_fields(self, dasquery):
        "Prepare fields to extract from MongoDB"
        fields     = dasquery.mongo_query.get('fields', None)
        if  fields == ['records']:
            fields = None # look-up all records
        filters    = dasquery.filters
        cond       = {}
        if  filters:
            new_fields = []
            for dasfilter in filters:
                if  dasfilter == 'unique':
                    continue
#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASParserDB(object):
    """
    Caching layer for the PLY parser.
    """
    def __init__(self, config):
        self.verbose  = config['verbose']
        self.logger   = PrintManager('DASParserDB', self.verbose)
        self.dburi    = config['mongodb']['dburi']
        self.dbname   = config['parserdb']['dbname']
        self.sizecap  = config['parserdb'].get('sizecap', 5*1024*1024)
        self.colname  = config['parserdb']['collname']
        
        msg = "DASParserCache::__init__ %[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)
        
        self.col = None
        self.create_db()

    def create_db(self):
        """
        Create db collection
        """
        conn = db_connection(self.dburi)
        dbn  = conn[self.dbname]
        if  self.colname not in dbn.collection_names():
            dbn.create_collection(self.colname, capped=True, size=self.sizecap)
        self.col = dbn[self.colname]

    def lookup_query(self, rawtext):
        """
        Check the parser cache for a given rawtext query.
        Search is done with the hash of this string.
        
        Returns a tuple (status, value) for the cases
        (PARSERCACHE_VALID, mongo_query) - valid query found
        (PARSERCACHE_INVALID, error) - error message for invalid query
        (PARSERCACHE_NOTFOUND, None) - not in the cache
        """
        result = self.col.find_one({'hash':genkey(rawtext)},
                        fields=['query', 'error'])

        if result and result['query']:
            if self.verbose:
                self.logger.debug("DASParserCache: found valid %s->%s" %\
                                  (rawtext, result['query']))
            
            query = decode_mongo_query(result['query'])
            return (PARSERCACHE_VALID, query)
        elif result and result['error']:
            if self.verbose:
                self.logger.debug("DASParserCache: found invalid %s->%s" %\
                                  (rawtext, result['error']))
            return (PARSERCACHE_INVALID, result['error'])
        else:
            if self.verbose:
                self.logger.debug("DASParserCache: not found %s" %\
                                  (rawtext))
            return (PARSERCACHE_NOTFOUND, None)

    def insert_valid_query(self, rawtext, query):
        "Insert a query that was successfully transformed"	
        self._insert_query(rawtext, query, None)

    def insert_invalid_query(self, rawtext, error):
        "Insert the error message for an invalid query"
        self._insert_query(rawtext, None, error)

    def _insert_query(self, rawtext, query, error):
        """Internal method to insert a query"""
        if  self.verbose:
            self.logger.debug("DASParserCache: insert %s->%s/%s" %\
	                          (rawtext, query, error))
        # since MongoDB does not support insertion of $ sign in queries
        # we need to encode inserted query
        if  query:
            encquery = encode_mongo_query(query)
        else:
            encquery = ""
        self.col.insert({'raw':rawtext, 'hash':genkey(rawtext),
                         'query':encquery, 'error':str(error)})

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASKeyLearning(object):
    """
    This class manages DAS key-learning DB.

    Key-learning is an intermittent process (triggered infrequently
    by a task running in the analytics framework), which involves
    searching through the raw cache for (a subset of but with
    maximum primary key coverage) all output documents, generating
    the set of all data members (in a dotted-dict fashion) and storing
    those as primary-key:data-member records (with an associated
    last-updated-time).

    """
    def __init__(self, config):
        self.verbose  = config['verbose']
        self.logger   = PrintManager('DASKeyLearning', self.verbose)
        self.services = config['services']
        self.dburi    = config['mongodb']['dburi']
        self.dbname   = config['keylearningdb']['dbname']
        self.colname  = config['keylearningdb']['collname']

        self.mapping  = config['dasmapping']

        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)

        self.das_son_manipulator = DAS_SONManipulator()
        index_list = [('system', ASCENDING), ('urn', ASCENDING), \
                ('members', ASCENDING), ('stems', ASCENDING)]
        create_indexes(self.col, index_list)

    @property
    def col(self):
        "col property provides access to DAS keylearning collection"
        conn = db_connection(self.dburi)
        mdb  = conn[self.dbname]
        colnames = mdb.collection_names()
        if  not colnames or self.colname not in colnames:
            try:
                mdb.create_collection(self.colname)
            except OperationFailure:
                pass
        mdb.add_son_manipulator(self.das_son_manipulator)
        return mdb[self.colname]

    def add_record(self, dasquery, rec):
        """
        Add/update to keylearning DB keys/attributes from given record.
        To do so, we parse it and call add_members method.
        """
        if  not ('das' in rec and 'system' in rec['das']):
            return
        das = rec['das']
        if  'system' not in das or 'api' not in das or 'primary_key' not in das:
            return
        systems = das['system']
        apis = das['api']
        pkey = das['primary_key'].split('.')[0]
        data = rec.get(pkey, [])
        members = dict_members(data, pkey)
        for srv, api in zip(systems, apis):
            self.add_members(srv, api, members)
        # insert new record for query patern
        fields = dasquery.mongo_query.get('fields', [])
        if  fields:
            for field in fields:
                if  field in DAS_RECORD_KEYS:
                    continue
                new_members = [m for m in dict_members(rec[field], field) if m]
                members += new_members
        for attr in members:
            spec = {'member': attr}
            doc = {'query_pat': dasquery.query_pat}
            self.col.update(spec, {'$addToSet': doc}, upsert=True)

    def add_members(self, system, urn, members):
        """
        Add a list of data members for a given API (system, urn, url),
        and generate, which are stored as separate records.
        """
        msg = "system=%s, urn=%s, members=%s)" % (system, urn, members)
        self.logger.info(msg)

        result = self.col.find_one({'system': system, 'urn': urn})
        if result:
            self.col.update({'_id': ObjectId(result['_id'])},
                            {'$addToSet': {'members': {'$each': members}}})
        else:
            keys = self.mapping.api2daskey(system, urn)
            self.col.insert({'system': system,
                             'urn': urn,
                             'keys': keys,
                             'members': members})

        for member in members:
            if not self.col.find_one({'member': member}):
                self.col.insert({'member': member,
                                 'stems': stem(member)})

    def text_search(self, text):
#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class KeyLearning(object):
    """
    This is the asynchronous part of the key-learning system, intended
    to run probably not much more than daily once the key learning DB is
    filled.
    
    This searches through the DAS raw cache for all API output records,
    recording at least `redundancy` das_ids for each primary_key found.
    
    These das_ids are then used to fetch the query record, which records
    the API system and urn of each of the records in question.
    
    These documents are then processed to extract all the unique member
    names they contained, which are then injected into the DAS keylearning
    system.
    """
    task_options = [{'name':'redundancy', 'type':'int', 'default':2,
                     'help':'Number of records to examine per DAS primary key'}]
    def __init__(self, **kwargs):
        self.logger = PrintManager('KeyLearning', kwargs.get('verbose', 0))
        self.das = kwargs['DAS']
        self.redundancy = kwargs.get('redundancy', 2)
        
        
    def __call__(self):
        "__call__ implementation"
        self.das.rawcache.remove_expired("cache")
        
        autodeque = lambda: collections.deque(maxlen=self.redundancy)
        found_ids = collections.defaultdict(autodeque)
        
        self.logger.info("finding das_ids")
        for doc in self.das.rawcache.col.find(\
            {'das.empty_record': 0, 'das.primary_key': {'$exists': True}},
            fields=['das.primary_key', 'das_id']):
            found_ids[doc['das']['primary_key']].append(doc['das_id'])
        
        hit_ids = set()
        
        self.logger.info("found %s primary_keys" % len(found_ids))
        
        for key in found_ids:
            self.logger.info("primary_key=%s" % key)
            for das_id in found_ids[key]:
                if not das_id in hit_ids:
                    self.logger.info("das_id=%s" % das_id)
                    hit_ids.add(das_id)
                    doc = self.das.rawcache.col.find_one(\
                        {'_id': ObjectId(das_id)})
                    if doc:
                        self.process_query_record(doc)
                    else:
                        self.logger.warning(\
                        "no record found for das_id=%s" % das_id)
        return {}
    
    def process_query_record(self, doc):
        """
        Process a rawcache document, extracting the called
        system, urn and url, then looking up the individual data records.
        """
        das_id = str(doc['_id'])
        systems = doc['das']['system']
        urns = doc['das']['urn']
        
        result = self.das.rawcache.find_records(das_id)        
        
        if len(systems)==len(urns) and len(systems)==result.count():
            for i, record in enumerate(result):
                self.process_document(systems[i], urns[i], record)
        else:
            self.logger.warning("got inconsistent system/urn/das_id length")
            
            
    def process_document(self, system, urn, doc):
        """
        Process a rawcache document record, finding all the unique
        data members and inserting them into the cache.
        """
        
        self.logger.info("%s::%s" % (system, urn))
        members = set()
        for key in doc.keys():
            if not key in ('das', '_id', 'das_id'):
                members |= self._process_document_recursor(doc[key], key)
        
        self.das.keylearning.add_members(system, urn, list(members))
        
    def _process_document_recursor(self, doc, prefix):
        """
        Recurse through a nested data structure, finding all
        the unique endpoint names. Lists are iterated over but do
        not add anything to the prefix, eg
        
        a: {b: 1, c: {d: 1, e: 1}, f: [{g: 1}, {h: 1}]} ->
        a.b, a.c.d, a.c.e, a.f.g, a.f.h
        
        (although normally we would expect each member of a list to
        have the same structure)
        """
#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]

#.........这里部分代码省略.........
        self.service_keys = {}
        self.service_parameters = {}
        for name in self.systems:
            skeys = list(getattr(self, name).keys())
            self.service_keys[getattr(self, name).name] = skeys
            sparams = getattr(self, name).parameters()
            self.service_parameters[getattr(self, name).name] = sparams

        self.service_keys["special"] = das_special_keys()
        self.dasconfig = dasconfig
        das_timer("DASCore::init", self.verbose)

    def keys(self):
        """
        Return map of data service keys
        """
        return self.service_keys

    def das_keys(self):
        """
        Return map of data service keys
        """
        _keys = ["records"]
        for values in self.service_keys.values():
            for key in values:
                if key not in _keys:
                    _keys.append(key)
        return _keys

    def result(self, query, idx=0, limit=None):
        """
        Get results either from cache or from explicit call
        """
        self.logger.info("input query=%s" % query)
        results = []
        dasquery = DASQuery(query)
        query = dasquery.mongo_query
        # check if we have any service which cover the query
        # otherwise decompose it into list of queries
        service_map = dasquery.service_apis_map()
        if not service_map:
            msg = "no APIs found to answer input query, will decompose it"
            self.logger.info(msg)
            skeys = query["fields"]
            if not skeys:
                skeys = []
            for key in skeys:
                newquery = DASQuery(dict(fields=[key], spec=query["spec"]))
                self.call(newquery)  # process query
        else:
            self.call(dasquery)  # process query

        # lookup provided query in a cache
        if not self.noresults:
            results = self.get_from_cache(dasquery, idx, limit)
        return results

    def remove_from_cache(self, dasquery):
        """
        Delete in cache entries about input query
        """
        self.rawcache.remove_from_cache(dasquery)

    def get_status(self, dasquery):
        """
        Look-up status of provided query in a cache.

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]

#.........这里部分代码省略.........
        self.service_keys = {}
        self.service_parameters = {}
        for name in self.systems: 
            skeys = getattr(self, name).keys()
            self.service_keys[getattr(self, name).name] = skeys
            sparams = getattr(self, name).parameters()
            self.service_parameters[getattr(self, name).name] = sparams

        self.service_keys['special'] = das_special_keys()
        self.dasconfig = dasconfig
        das_timer('DASCore::init', self.verbose)

    def keys(self):
        """
        Return map of data service keys
        """
        return self.service_keys

    def das_keys(self):
        """
        Return map of data service keys
        """
        _keys = ['records']
        for values in self.service_keys.values():
            for key in values:
                if  key not in _keys:
                    _keys.append(key)
        return _keys

    def result(self, query, idx=0, limit=None):
        """
        Get results either from cache or from explicit call
        """
        self.logger.info('input query=%s' % query)
        results = []
        dasquery = DASQuery(query, mongoparser=self.mongoparser)
        dasquery.add_to_analytics()
        query    = dasquery.mongo_query
        # check if we have any service which cover the query
        # otherwise decompose it into list of queries
        service_map = dasquery.service_apis_map()
        if  not service_map:
            msg  = 'no APIs found to answer input query, will decompose it'
            self.logger.info(msg)
            skeys = query['fields']
            if  not skeys:
                skeys = []
            for key in skeys:
                newquery = DASQuery(dict(fields=[key], spec=query['spec']),
                                        mongoparser=self.mongoparser)
                self.call(newquery) # process query
        else:
            self.call(dasquery) # process query

        # lookup provided query in a cache
        if  not self.noresults:
            results = self.get_from_cache(dasquery, idx, limit)
        return results

    def remove_from_cache(self, dasquery):
        """
        Delete in cache entries about input query
        """
        self.rawcache.remove_from_cache(dasquery)

    def get_status(self, dasquery):

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASMapping(object):
    """
    This class manages DAS mapping DB.
    """

    def __init__(self, config):
        self.verbose = config["verbose"]
        self.logger = PrintManager("DASMapping", self.verbose)
        self.services = config["services"]
        self.dburi = config["mongodb"]["dburi"]
        self.dbname = config["mappingdb"]["dbname"]
        self.colname = config["mappingdb"]["collname"]
        self.map_test = config.get("map_test", True)
        self.main_dbs = config["das"].get("main_dbs", "dbs")
        self.dbsinsts = config["das"].get("dbs_instances", [])

        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)

        self.init()
        self.on_reload = Event()

        # Monitoring thread which performs auto-reconnection to MongoDB
        thname = "mappingdb_monitor"
        sleep = 5
        reload_time = config["mappingdb"].get("reload_time", 86400)
        reload_time_bad_maps = config["mappingdb"].get("reload_time_bad_maps", 120)
        start_new_thread(
            thname,
            db_monitor,
            (self.dburi, self.init, sleep, self.load_maps, reload_time, self.check_maps, reload_time_bad_maps),
        )

        self.daskeyscache = {}  # to be filled at run time
        self.systems = []  # to be filled at run time
        self.dasmapscache = {}  # to be filled at run time
        self.keymap = {}  # to be filled at run time
        self.presentationcache = {}  # to be filled at run time
        self.reverse_presentation = {}  # to be filled at run time
        self.notationcache = {}  # to be filled at run time
        self.diffkeycache = {}  # to be filled at run time
        self.apicache = {}  # to be filled at run time
        self.dbs_global_url = None  # to be determined at run time
        self.dbs_inst_names = None  # to be determined at run time
        self.load_maps(notify=False)

    @property
    def col(self):
        "Return MongoDB collection object"
        conn = db_connection(self.dburi)
        dbc = conn[self.dbname]
        col = dbc[self.colname]
        return col

    # ===============
    # Management APIs
    # ===============
    def load_maps(self, notify=True):
        "Helper function to reload DAS maps"
        self.init_dasmapscache()
        self.init_notationcache()
        self.init_presentationcache()
        self.systems = None  # re-initialize DAS system list
        self.list_systems()
        self.dbs_global_url = None  # re-initialize DAS dbs global url
        self.dbs_url()
        self.dbs_inst_names = None  # re-initialize DAS dbs instances
        self.dbs_instances()

        if notify:
            self.on_reload()

    def init_dasmapscache(self, records=[]):
        "Read DAS maps and initialize DAS API maps"
        if not records:
            spec = {"type": "service"}
            records = self.col.find(spec, exhaust=True)
        for row in records:
            if "urn" in row:
                api = row["urn"]
                srv = row["system"]
                for dmap in row["das_map"]:
                    for key, val in dmap.iteritems():
                        if key == "pattern":
                            pat = re.compile(val)
                            dmap[key] = pat
                key = (row["system"], row["urn"])
                self.dasmapscache[key] = row

    def init_notationcache(self):
        """
        Initialize notation cache by reading notations.
        """
        for system, notations in self.notations().iteritems():
            for row in notations:
                key = system, row["api_output"]
                if key in self.notationcache:
                    self.notationcache[key] += [(row["api"], row["rec_key"])]
                else:
                    self.notationcache[key] = [(row["api"], row["rec_key"])]
#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]

#.........这里部分代码省略.........

        TODO: it would be nice to implement clustering algorithm
        which will pass only items with higher weight.
        """
        if not self.allow_wildcarding:
            for key in items.keys():
                if '*' in key:
                    del items[key]
        return items
    
    def mutate_items(self, items):
        """
        Mutate items for task generation. This is a last call in
        selection chain.
        """
        if self.find_supersets:
            new_keys = self.get_superset_keys(items.keys())
            return dict([(k, items.get(k, 0)) for k in new_keys])
        else:
            return items
    
    def get_superset_keys(self, keys):
        """
        For multiple keys, try and identify an existing queries for
        wildcard supersets of the keys, and reduce the keylist appropriately.
        Important to note, this only uses existing queries (won't try
        and make new ones).
        """
        superset_cache = {}
        
        keys = set(keys)
        change_made = True
        while change_made:
            change_made = False
            for key in list(keys):
                if key in superset_cache:
                    superset_keys = superset_cache[key]
                else:
                    try:
                        superset_keys = \
                            sorted([k for k in \
                            self.das.rawcache.get_superset_keys(self.key, key)],\
                            key=len)
                    except:
                        superset_keys = []
                if superset_keys:
                    super_key = superset_keys[0]
                    for key in keys:
                        if fnmatch.fnmatch(super_key, key):
                            keys.remove(key)
                            keys.add(super_key)
                            change_made = True
                    if change_made:
                        break
        return keys
    
    def get_query_expiry(self, dasquery):
        """
        Extract analytics apicall the expire timestamp for given query.
        If query is not found in DAS cache, it invokes DAS call.
        """
        err_return = time.time() + (2*self.preempt)
        try:
            if not self.das.rawcache.incache(dasquery):
                try:
                    self.das.call(dasquery, add_to_analytics=False)
                except Exception as err:
                    print "\n### FAIL input query=%s, err=%s" \
                            % (dasquery, str(err))
                    raise err
            expiries = [result.get('apicall', {}).get('expire', 0) for result in \
                            self.das.analytics.list_apicalls(qhash=dasquery.qhash)]
            if  not expiries:
                return err_return
            return min(expiries)
        except:
            return err_return
              
    def make_one_summary(self, start, finish):
        "Actually make the summary"
        keys = collections.defaultdict(int)
        try:
            queries = self.das.analytics.list_queries(key=self.key,
                                                      after=start,
                                                      before=finish)
        except:
            queries = []
        for query in queries:
            count = len(filter(lambda t: t>=start and t<=finish, 
                               query['times']))
            for spec in query['mongoquery']['spec']:
                if spec['key'] == self.key:
                    keys[spec['value']] += count
        if keys:
            self.logger.info("Found %s queries in %s->%s" \
                             % (len(keys), start, finish))
        else:
            self.logger.info("Found no queries in %s->%s" \
                             % (start, finish))
        return keys

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASMapping(object):
    """
    This class manages DAS mapping DB.
    """

    def __init__(self, config):
        self.verbose = config["verbose"]
        self.logger = PrintManager("DASMapping", self.verbose)
        self.services = config["services"]
        self.dburi = config["mongodb"]["dburi"]
        self.dbname = config["mappingdb"]["dbname"]
        self.colname = config["mappingdb"]["collname"]

        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)

        self.create_db()

        self.keymap = {}  # to be filled at run time
        self.presentationcache = {}  # to be filled at run time
        self.reverse_presentation = {}  # to be filled at run time
        self.notationcache = {}  # to be filled at run time
        self.diffkeycache = {}  # to be filled at run time
        self.apicache = {}  # to be filled at run time
        self.apiinfocache = {}  # to be filled at run time
        self.init_notationcache()
        self.init_presentationcache()

    # ===============
    # Management APIs
    # ===============
    def init_notationcache(self):
        """
        Initialize notation cache by reading notations.
        """
        for system, notations in self.notations().iteritems():
            for row in notations:
                key = system, row["notation"]
                if self.notationcache.has_key(key):
                    self.notationcache[key] += [(row["api"], row["map"])]
                else:
                    self.notationcache[key] = [(row["api"], row["map"])]

    def init_presentationcache(self):
        """
        Initialize presentation cache by reading presentation map.
        """
        query = {"presentation": {"$ne": None}}
        data = self.col.find_one(query)
        if data:
            self.presentationcache = data["presentation"]
            for daskey, uilist in self.presentationcache.iteritems():
                for row in uilist:
                    link = None
                    if row.has_key("link"):
                        link = row["link"]
                    if row.has_key("diff"):
                        self.diffkeycache[daskey] = row["diff"]
                    tdict = {daskey: {"mapkey": row["das"], "link": link}}
                    if self.reverse_presentation.has_key(row["ui"]):
                        self.reverse_presentation[row["ui"]].update(tdict)
                    else:
                        self.reverse_presentation[row["ui"]] = {daskey: {"mapkey": row["das"], "link": link}}

    def create_db(self):
        """
        Establish connection to MongoDB back-end and create DB.
        """
        self.conn = db_connection(self.dburi)
        self.db = self.conn[self.dbname]
        self.col = self.db[self.colname]

    def delete_db(self):
        """
        Delete mapping DB in MongoDB back-end.
        """
        self.conn.drop_database(self.dbname)

    def delete_db_collection(self):
        """
        Delete mapping DB collection in MongoDB.
        """
        self.db.drop_collection(self.colname)

    def check_maps(self):
        """
        Check if there are records in Mapping DB
        """
        return self.col.count()

    def remove(self, spec):
        """
        Remove record in DAS Mapping DB for provided Mongo spec.
        """
        self.col.remove(spec)

    def add(self, record):
        """
        Add new record into mapping DB. Example of URI record

#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class HotspotBase(object):
    """
    This is a base-class for periodically-running
    analyzers that want to examine the moving average
    of some key->counter map, and pick the top few
    for further attention.

    DASQueries are extracted from analytics DB. The selected items
    are passed to generate_task callback implemented in subclasses.
    It look-up DAS query expiration timestamp and if necessary
    calls DAS to get it (along with results of the query).
    """
    def __init__(self, **kwargs):
        self.logger = PrintManager('HotspotBase', kwargs.get('verbose', 0))
        self.das = kwargs['DAS']
        self.fraction = float(kwargs.get('fraction', 0.15))
        self.mode = kwargs.get('mode','calls').lower()
        self.period = int(kwargs.get('period', 86400*30))
        self.interval = kwargs['interval']
        self.allowed_gap = int(kwargs.get('allowed_gap', 3600))
        self.identifier = kwargs['identifier']

    def __call__(self):
        """
        Perform a hotspot-like analysis. Subclasses shouldn't
        need to reimplement this method.

        We start with building selection chain. It consists of
        analytics summaries -> items -> preselected items ->
        mutable items. The final set of items is passed to
        task generation step (implemented in subclasses).
        The final report is generated and returned back.
        """

        epoch_end = time.time()
        epoch_start = epoch_end - self.period

        summaries = self.get_summaries(epoch_start, epoch_end)
        self.logger.info("Got %s summaries" % len(summaries))

        items = self.get_all_items(summaries)
        self.logger.info("Got %s items" % len(items))

        items = self.preselect_items(items)
        self.logger.info("Preselected to %s items" % len(items))

        items = self.select_items(items)
        self.logger.info("Selected %s items (%s:%s)" \
                         % (len(items), self.mode, self.fraction))

        items = self.mutate_items(items)
        self.logger.info("Mutated to %s items" % len(items))

        retval = {'mode': self.mode,
                  'fraction': self.fraction,
                  'epoch_start': epoch_start,
                  'epoch_end': epoch_end,
                  'summaries': len(summaries),
                  'selected': dict(items).items()}

        new_tasks = []
        failed_items = []
        for item, count in items.items():
            try:
                self.logger.info("Generating task for %s" % item)
                for task in \
                    self.generate_task(item, count, epoch_start, epoch_end):
                    new_tasks.append(task)
            except Exception as exc:
                failed_items.append((item, count, str(exc)))
        retval['new_tasks'] = new_tasks
        retval['failed_items'] = failed_items

        retval.update(self.report())

        return retval

    def generate_task(self, item, count, epoch_start, epoch_end):
        """
        For the given selected key, generate an appropriate task
        dictionary as understood by taskscheduler.

        Should be a generator or return an iterable
        """
        raise NotImplementedError

    def report(self):
        """
        Generate some extra keys to go in the job report, if desired.
        """
        return {}

    def preselect_items(self, items):
        """
        This is a part of selection chain.

        Optionally, preselect the items for consideration.
        A subclass wishing to exclude certain key types could
        do so here (but could also do so in make_one_summary).

#.........这里部分代码省略.........

# 需要导入模块: from DAS.utils.logger import PrintManager [as 别名]
# 或者: from DAS.utils.logger.PrintManager import info [as 别名]
class DASAnalytics(object):
    """
    DAS analytics DB manager.
    """
    def __init__(self, config):
        self.verbose = config['verbose']
        self.logger  = PrintManager('DASAnalytics', self.verbose)
        self.dburi   = config['mongodb']['dburi']
        self.dbname  = config['analyticsdb']['dbname']        
        self.colname = config['analyticsdb']['collname']
        self.history = config['analyticsdb']['history']
        msg = "%[email protected]%s" % (self.dburi, self.dbname)
        self.logger.info(msg)
        self.create_db()

    def create_db(self):
        """
        Create analytics DB in MongoDB back-end.
        """
        self.conn = db_connection(self.dburi)
        database  = self.conn[self.dbname]
        das_son_manipulator = DAS_SONManipulator()
        database.add_son_manipulator(das_son_manipulator)
        self.col  = database[self.colname]
#        if  self.dbname not in self.conn.database_names():
#            capped_size = 104857600
#            options   = {'capped':True, 'size': capped_size}
#            database  = self.conn[self.dbname]
#            database.create_collection('self.colname', **options)
#            print "####CREATE CAPPED ANALYTICS"
#        self.col  = self.conn[self.dbname][self.colname] 

    def delete_db(self):
        """
        Delete analytics DB in MongoDB back-end.
        """
        self.conn.drop_database(self.dbname)

    def delete_db_collection(self):
        """
        Delete analytics DB collection in MongoDB.
        """
        self.conn.drop_collection(self.colname)

    def add_query(self, query, mongoquery):
        """
        Add DAS-QL/MongoDB-QL queries into analytics.
        
        A unique record is contained for each (qhash, dhash) pair.
        For each an array of call-times is contained.
        """
        if  isinstance(mongoquery, dict):
            mongoquery = encode_mongo_query(mongoquery)
        msg = 'query=%s, mongoquery=%s' % (query, mongoquery)
        self.logger.debug(msg)
        dhash = genkey(query)
        qhash = genkey(mongoquery)

        now = time.time()

        existing = self.col.find_one({'qhash': qhash, 'dhash': dhash})
        if existing:
            # check if times contains very old timestamps
            rec = self.col.find({'_id': ObjectId(existing['_id']), 
                                 'times':{'$lt' : now - self.history}})
            if  rec:
                self.col.update({'_id': ObjectId(existing['_id'])},
                    {'$pull': {'times': {'$lt' : now - self.history}}})
            # update times array with new timestamp
            self.col.update({'_id': ObjectId(existing['_id'])},
                            {'$push': {'times': now}})
        else:
            record = dict(query=query, mongoquery=mongoquery,
                        qhash=qhash, dhash=dhash, times=[now])
            self.col.insert(record)

        index = [('qhash', DESCENDING),
                 ('dhash', DESCENDING)]
        create_indexes(self.col, index)
        
    def clean_queries(self):
        """
        Standalone method to clean up expired call-times from query records,
        since otherwise only the active record is cleaned.
        
        This is too expensive to do with every operation, and mongodb
        does not allow multiple modifications to a single field in a single
        update operation (ie, we can't do $push and $pull in one update),
        so it should probably be done asynchronously at fixed intervals.
        """
        
        self.logger.debug('')
        
        now = time.time()
        
        #clean out the times array
        self.col.update({'times': {'$exists': True}},
                        {'$pull': {'times': {'$lt': now - self.history}}})
        #now delete any with no times
        self.col.remove({'times': {'$size': 0}})
#.........这里部分代码省略.........