Python Toolbox.log_cache_hit方法代码示例

# 需要导入模块: from ovs.dal.helpers import Toolbox [as 别名]
# 或者: from ovs.dal.helpers.Toolbox import log_cache_hit [as 别名]
 def load_foreign_relations(object_type):
     """
     This method will return a mapping of all relations towards a certain hybrid object type.
     The resulting mapping will be stored in volatile storage so it can be fetched faster
     """
     relation_key = 'ovs_relations_{0}'.format(object_type.__name__.lower())
     volatile = VolatileFactory.get_client()
     relation_info = volatile.get(relation_key)
     if relation_info is None:
         Toolbox.log_cache_hit('relations', False)
         relation_info = {}
         hybrid_structure = HybridRunner.get_hybrids()
         for class_descriptor in hybrid_structure.values():  # Extended objects
             cls = Descriptor().load(class_descriptor).get_object()
             for relation in cls._relations:
                 if relation.foreign_type is None:
                     remote_class = cls
                 else:
                     identifier = Descriptor(relation.foreign_type).descriptor['identifier']
                     if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
                         remote_class = Descriptor().load(hybrid_structure[identifier]).get_object()
                     else:
                         remote_class = relation.foreign_type
                 itemname = remote_class.__name__
                 if itemname == object_type.__name__:
                     relation_info[relation.foreign_key] = {'class': Descriptor(cls).descriptor,
                                                            'key': relation.name,
                                                            'list': not relation.onetoone}
         volatile.set(relation_key, relation_info)
     else:
         Toolbox.log_cache_hit('relations', True)
     return relation_info

# 需要导入模块: from ovs.dal.helpers import Toolbox [as 别名]
# 或者: from ovs.dal.helpers.Toolbox import log_cache_hit [as 别名]
    def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid):
        """
        This method will get a DataList for a relation.
        On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
        update all relations in the mean time.
        """

        # Example:
        # * remote_class = vDisk
        # * remote_key = vmachine
        # * own_class = vMachine
        # * own_key = vdisks
        # Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
        # * own_guid = this vMachine object's guid

        volatile = VolatileFactory.get_client()
        own_name = own_class.__name__.lower()
        datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False)
        reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)

        # Check whether the requested information is available in cache
        reverse_index = volatile.get(reverse_key)
        if reverse_index is not None and own_key in reverse_index:
            Toolbox.log_cache_hit('datalist', True)
            datalist.data = reverse_index[own_key]
            datalist.from_cache = True
            return datalist

        Toolbox.log_cache_hit('datalist', False)
        mutex = VolatileMutex('reverseindex')
        remote_name = remote_class.__name__.lower()
        blueprint_object = remote_class()  # vDisk object
        foreign_guids = {}

        remote_namespace = blueprint_object._namespace
        for relation in blueprint_object._relations:  # E.g. vmachine or vpool relation
            if relation.foreign_type is None:
                classname = remote_name
                foreign_namespace = blueprint_object._namespace
            else:
                classname = relation.foreign_type.__name__.lower()
                foreign_namespace = relation.foreign_type()._namespace
            if classname not in foreign_guids:
                foreign_guids[classname] = DataList.get_pks(foreign_namespace, classname)
            try:
                mutex.acquire(60)
                for foreign_guid in foreign_guids[classname]:
                    reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, foreign_guid)
                    reverse_index = volatile.get(reverse_key)
                    if reverse_index is None:
                        reverse_index = {}
                    if relation.foreign_key not in reverse_index:
                        reverse_index[relation.foreign_key] = []
                        volatile.set(reverse_key, reverse_index)
            finally:
                mutex.release()
        remote_keys = DataList.get_pks(remote_namespace, remote_name)
        for guid in remote_keys:
            try:
                instance = remote_class(guid)
                for relation in blueprint_object._relations:  # E.g. vmachine or vpool relation
                    if relation.foreign_type is None:
                        classname = remote_name
                    else:
                        classname = relation.foreign_type.__name__.lower()
                    key = getattr(instance, '{0}_guid'.format(relation.name))
                    if key is not None:
                        try:
                            mutex.acquire(60)
                            reverse_index = volatile.get('ovs_reverseindex_{0}_{1}'.format(classname, key))
                            if reverse_index is None:
                                reverse_index = {}
                            if relation.foreign_key not in reverse_index:
                                reverse_index[relation.foreign_key] = []
                            if guid not in reverse_index[relation.foreign_key]:
                                if instance.updated_on_datastore():
                                    raise ConcurrencyException()
                                reverse_index[relation.foreign_key].append(guid)
                                volatile.set('ovs_reverseindex_{0}_{1}'.format(classname, key), reverse_index)
                        finally:
                            mutex.release()
            except ObjectNotFoundException:
                pass
            except ConcurrencyException:
                pass

        try:
            mutex.acquire(60)
            reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
            reverse_index = volatile.get(reverse_key)
            if reverse_index is None:
                reverse_index = {}
            if own_key not in reverse_index:
                reverse_index[own_key] = []
                volatile.set(reverse_key, reverse_index)
            datalist.data = reverse_index[own_key]
            datalist.from_cache = False
        finally:
            mutex.release()
        return datalist

# 需要导入模块: from ovs.dal.helpers import Toolbox [as 别名]
# 或者: from ovs.dal.helpers.Toolbox import log_cache_hit [as 别名]
    def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
        """
        Loads an object with a given guid. If no guid is given, a new object
        is generated with a new guid.
        * guid: The guid indicating which object should be loaded
        * datastoreWins: Optional boolean indicating save conflict resolve management.
        ** True: when saving, external modified fields will not be saved
        ** False: when saving, all changed data will be saved, regardless of external updates
        ** None: in case changed field were also changed externally, an error will be raised
        """

        # Initialize super class
        super(DataObject, self).__init__()

        # Initialize internal fields
        self._frozen = False
        self._datastore_wins = datastore_wins
        self._guid = None             # Guid identifier of the object
        self._original = {}           # Original data copy
        self._metadata = {}           # Some metadata, mainly used for unit testing
        self._data = {}               # Internal data storage
        self._objects = {}            # Internal objects storage

        # Initialize public fields
        self.dirty = False
        self.volatile = volatile

        # Worker fields/objects
        self._classname = self.__class__.__name__.lower()
        self._namespace = 'ovs_data'   # Namespace of the object
        self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname))
        self._mutex_reverseindex = VolatileMutex('reverseindex')

        # Rebuild _relation types
        hybrid_structure = HybridRunner.get_hybrids()
        for relation in self._relations:
            if relation.foreign_type is not None:
                identifier = Descriptor(relation.foreign_type).descriptor['identifier']
                if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
                    relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()

        # Init guid
        self._new = False
        if guid is None:
            self._guid = str(uuid.uuid4())
            self._new = True
        else:
            guid = str(guid).lower()
            if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None:
                self._guid = str(guid)
            else:
                raise ValueError('The given guid is invalid: {0}'.format(guid))

        # Build base keys
        self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid)

        # Worker mutexes
        self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))

        # Load data from cache or persistent backend where appropriate
        self._volatile = VolatileFactory.get_client()
        self._persistent = PersistentFactory.get_client()
        self._metadata['cache'] = None
        if self._new:
            self._data = {}
        else:
            self._data = self._volatile.get(self._key)
            if self._data is None:
                Toolbox.log_cache_hit('object_load', False)
                self._metadata['cache'] = False
                try:
                    self._data = self._persistent.get(self._key)
                except KeyNotFoundException:
                    raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
                        self.__class__.__name__, self._guid
                    ))
            else:
                Toolbox.log_cache_hit('object_load', True)
                self._metadata['cache'] = True

        # Set default values on new fields
        for prop in self._properties:
            if prop.name not in self._data:
                self._data[prop.name] = prop.default
            self._add_property(prop)

        # Load relations
        for relation in self._relations:
            if relation.name not in self._data:
                if relation.foreign_type is None:
                    cls = self.__class__
                else:
                    cls = relation.foreign_type
                self._data[relation.name] = Descriptor(cls).descriptor
            self._add_relation_property(relation)

        # Add wrapped properties
        for dynamic in self._dynamics:
            self._add_dynamic_property(dynamic)

#.........这里部分代码省略.........

# 需要导入模块: from ovs.dal.helpers import Toolbox [as 别名]
# 或者: from ovs.dal.helpers.Toolbox import log_cache_hit [as 别名]
    def _load(self):
        """
        Tries to load the result for the given key from the volatile cache, or executes the query
        if not yet available. Afterwards (if a key is given), the result will be (re)cached
        """
        self.data = self._volatile.get(self._key) if self._key is not None else None
        if self.data is None:
            # The query should be a dictionary:
            #     {'object': Disk,  # Object on which the query should be executed
            #      'data'  : DataList.select.XYZ,  # The requested result
            #      'query' : <query>}  # The actual query
            # Where <query> is a query(group) dictionary:
            #     {'type' : DataList.where_operator.ABC,  # Whether the items should be AND/OR
            #      'items': <items>}  # The items in the group
            # Where the <items> is any combination of one or more <filter> or <query>
            # A <filter> tuple example:
            #     (<field>, DataList.operator.GHI, <value>)  # For example EQUALS
            # The field is any property you would also find on the given object. In case of
            # properties, you can dot as far as you like. This means you can combine AND and OR
            # in any possible combination

            Toolbox.log_cache_hit('datalist', False)
            hybrid_structure = HybridRunner.get_hybrids()

            items = self._query['query']['items']
            query_type = self._query['query']['type']
            query_data = self._query['data']
            query_object = self._query['object']
            query_object_id = Descriptor(query_object).descriptor['identifier']
            if query_object_id in hybrid_structure and query_object_id != hybrid_structure[query_object_id]['identifier']:
                query_object = Descriptor().load(hybrid_structure[query_object_id]).get_object()

            invalidations = {query_object.__name__.lower(): ['__all']}
            DataList._build_invalidations(invalidations, query_object, items)

            for class_name in invalidations:
                key = '{0}_{1}'.format(DataList.cachelink, class_name)
                mutex = VolatileMutex('listcache_{0}'.format(class_name))
                try:
                    mutex.acquire(60)
                    cache_list = Toolbox.try_get(key, {})
                    current_fields = cache_list.get(self._key, [])
                    current_fields = list(set(current_fields + ['__all'] + invalidations[class_name]))
                    cache_list[self._key] = current_fields
                    self._volatile.set(key, cache_list)
                    self._persistent.set(key, cache_list)
                finally:
                    mutex.release()

            self.from_cache = False
            namespace = query_object()._namespace
            name = query_object.__name__.lower()
            guids = DataList.get_pks(namespace, name)

            if query_data == DataList.select.COUNT:
                self.data = 0
            else:
                self.data = []

            for guid in guids:
                try:
                    instance = query_object(guid)
                    if query_type == DataList.where_operator.AND:
                        include = self._exec_and(instance, items)
                    elif query_type == DataList.where_operator.OR:
                        include = self._exec_or(instance, items)
                    else:
                        raise NotImplementedError('The given operator is not yet implemented.')
                    if include:
                        if query_data == DataList.select.COUNT:
                            self.data += 1
                        elif query_data == DataList.select.GUIDS:
                            self.data.append(guid)
                        else:
                            raise NotImplementedError('The given selector type is not implemented')
                except ObjectNotFoundException:
                    pass

            if 'post_query' in DataList.test_hooks:
                DataList.test_hooks['post_query'](self)

            if self._key is not None and len(guids) > 0 and self._can_cache:
                invalidated = False
                for class_name in invalidations:
                    key = '{0}_{1}'.format(DataList.cachelink, class_name)
                    cache_list = Toolbox.try_get(key, {})
                    if self._key not in cache_list:
                        invalidated = True
                # If the key under which the list should be saved was already invalidated since the invalidations
                # were saved, the returned list is most likely outdated. This is OK for this result, but the list
                # won't get cached
                if invalidated is False:
                    self._volatile.set(self._key, self.data, 300 + randint(0, 300))  # Cache between 5 and 10 minutes
        else:
            Toolbox.log_cache_hit('datalist', True)
            self.from_cache = True
        return self