Python volatilefactory.VolatileFactory类代码示例

 def new_function(self, request, *args, **kwargs):
     """
     Wrapped function
     """
     now = time.time()
     key = 'ovs_api_limit_{0}.{1}_{2}'.format(
         f.__module__, f.__name__,
         request.META['HTTP_X_REAL_IP']
     )
     client = VolatileFactory.get_client()
     mutex = VolatileMutex(key)
     try:
         mutex.acquire()
         rate_info = client.get(key, {'calls': [],
                                      'timeout': None})
         active_timeout = rate_info['timeout']
         if active_timeout is not None:
             if active_timeout > now:
                 return HttpResponse, {'error_code': 'rate_limit_timeout',
                                       'error': 'Rate limit timeout ({0}s remaining)'.format(round(active_timeout - now, 2))}, 429
             else:
                 rate_info['timeout'] = None
         rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
         calls = len(rate_info['calls'])
         if calls > amount:
             rate_info['timeout'] = now + timeout
             client.set(key, rate_info)
             return HttpResponse, {'error_code': 'rate_limit_reached',
                                   'error': 'Rate limit reached ({0} in last {1}s)'.format(calls, per)}, 429
         client.set(key, rate_info)
     finally:
         mutex.release()
     return f(self, request, *args, **kwargs)

    def __init__(self, object_type, query, key=None):
        """
        Initializes a DataList class with a given key (used for optional caching) and a given query
        :param object_type: The type of the objects that have to be queried
        :param query: The query to execute
        :param key: A key under which the result must be cached
        """
        super(DataList, self).__init__()

        if key is not None:
            self._key = '{0}_{1}'.format(DataList.NAMESPACE, key)
        else:
            identifier = copy.deepcopy(query)
            identifier['object'] = object_type.__name__
            self._key = '{0}_{1}'.format(DataList.NAMESPACE, hashlib.sha256(json.dumps(identifier)).hexdigest())

        self._volatile = VolatileFactory.get_client()
        self._persistent = PersistentFactory.get_client()
        self._query = query
        self._can_cache = True
        self._object_type = object_type
        self._data = {}
        self._objects = {}
        self._guids = None
        self._executed = False
        self._shallow_sort = True

        self.from_cache = None

    def __init__(self, ip, port, credentials=None, verify=False, version="*", raw_response=False):
        """
        Initializes the object with credentials and connection information
        """
        if credentials is not None and len(credentials) != 2:
            raise RuntimeError(
                "Credentials should be None (no authentication) or a tuple containing client_id and client_secret (authenticated)"
            )
        self.ip = ip
        self.port = port
        self.client_id = credentials[0] if credentials is not None else None
        self.client_secret = credentials[1] if credentials is not None else None
        self._url = "https://{0}:{1}/api".format(ip, port)
        self._key = hashlib.sha256(
            "{0}{1}{2}{3}".format(self.ip, self.port, self.client_id, self.client_secret)
        ).hexdigest()
        self._token = None
        self._verify = verify
        self._version = version
        self._raw_response = raw_response
        try:
            from ovs.extensions.storage.volatilefactory import VolatileFactory

            self._volatile_client = VolatileFactory.get_client()
        except ImportError:
            self._volatile_client = None

    def sync_with_reality(storagerouter_guid=None, max_attempts=3):
        """
        Try to run sync_with_reality, retry in case of failure
         always run sync, as tasks calling this expect this to be sync
        :param storagerouter_guid:
        :return:
        """
        cache = VolatileFactory.get_client()
        mutex = VolatileMutex('ovs_disk_sync_with_reality_{0}'.format(storagerouter_guid))

        key = 'ovs_dedupe_sync_with_reality_{0}'.format(storagerouter_guid)
        attempt = 1
        while attempt < max_attempts:
            task_id = cache.get(key)
            if task_id:
                revoke(task_id)
            try:
                mutex.acquire(wait=120)
                return DiskController._sync_with_reality(storagerouter_guid)
            except Exception as ex:
                logger.warning('Sync with reality failed. {0}'.format(ex))
                attempt += 1
                time.sleep(attempt*30)
            finally:
                mutex.release()

        raise RuntimeError('Sync with reality failed after 3 attempts')

 def new_function(self, request, *args, **kwargs):
     """
     Wrapped function
     """
     now = time.time()
     key = 'ovs_api_limit_{0}.{1}_{2}'.format(
         f.__module__, f.__name__,
         request.META['HTTP_X_REAL_IP']
     )
     client = VolatileFactory.get_client()
     mutex = VolatileMutex(key)
     try:
         mutex.acquire()
         rate_info = client.get(key, {'calls': [],
                                      'timeout': None})
         active_timeout = rate_info['timeout']
         if active_timeout is not None:
             if active_timeout > now:
                 raise Throttled(wait=active_timeout - now)
             else:
                 rate_info['timeout'] = None
         rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
         calls = len(rate_info['calls'])
         if calls > amount:
             rate_info['timeout'] = now + timeout
             client.set(key, rate_info)
             raise Throttled(wait=timeout)
         client.set(key, rate_info)
     finally:
         mutex.release()
     return f(self, request, *args, **kwargs)

 def load_foreign_relations(object_type):
     """
     This method will return a mapping of all relations towards a certain hybrid object type.
     The resulting mapping will be stored in volatile storage so it can be fetched faster
     """
     relation_key = 'ovs_relations_{0}'.format(object_type.__name__.lower())
     volatile = VolatileFactory.get_client()
     relation_info = volatile.get(relation_key)
     if relation_info is None:
         Toolbox.log_cache_hit('relations', False)
         relation_info = {}
         hybrid_structure = HybridRunner.get_hybrids()
         for class_descriptor in hybrid_structure.values():  # Extended objects
             cls = Descriptor().load(class_descriptor).get_object()
             for relation in cls._relations:
                 if relation.foreign_type is None:
                     remote_class = cls
                 else:
                     identifier = Descriptor(relation.foreign_type).descriptor['identifier']
                     if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
                         remote_class = Descriptor().load(hybrid_structure[identifier]).get_object()
                     else:
                         remote_class = relation.foreign_type
                 itemname = remote_class.__name__
                 if itemname == object_type.__name__:
                     relation_info[relation.foreign_key] = {'class': Descriptor(cls).descriptor,
                                                            'key': relation.name,
                                                            'list': not relation.onetoone}
         volatile.set(relation_key, relation_info)
     else:
         Toolbox.log_cache_hit('relations', True)
     return relation_info

        def new_function(*args, **kwargs):
            """
            Wrapped function
            """
            request = _find_request(args)

            now = time.time()
            key = 'ovs_api_limit_{0}.{1}_{2}'.format(
                f.__module__, f.__name__,
                request.META['HTTP_X_REAL_IP']
            )
            client = VolatileFactory.get_client()
            with VolatileMutex(key):
                rate_info = client.get(key, {'calls': [],
                                             'timeout': None})
                active_timeout = rate_info['timeout']
                if active_timeout is not None:
                    if active_timeout > now:
                        logger.warning('Call {0} is being throttled with a wait of {1}'.format(key, active_timeout - now))
                        raise Throttled(wait=active_timeout - now)
                    else:
                        rate_info['timeout'] = None
                rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
                calls = len(rate_info['calls'])
                if calls > amount:
                    rate_info['timeout'] = now + timeout
                    client.set(key, rate_info)
                    logger.warning('Call {0} is being throttled with a wait of {1}'.format(key, timeout))
                    raise Throttled(wait=timeout)
                client.set(key, rate_info)
            return f(*args, **kwargs)

        def new_function(*args, **kwargs):
            """
            Wrapped function
            """
            request = _find_request(args)

            now = time.time()
            key = 'ovs_api_limit_{0}.{1}_{2}'.format(
                f.__module__, f.__name__,
                request.META['HTTP_X_REAL_IP']
            )
            client = VolatileFactory.get_client()
            with volatile_mutex(key):
                rate_info = client.get(key, {'calls': [],
                                             'timeout': None})
                active_timeout = rate_info['timeout']
                if active_timeout is not None:
                    if active_timeout > now:
                        logger.warning('Call {0} is being throttled with a wait of {1}'.format(key, active_timeout - now))
                        raise HttpTooManyRequestsException(error='rate_limit_timeout',
                                                           error_description='Rate limit timeout ({0}s remaining)'.format(round(active_timeout - now, 2)))
                    else:
                        rate_info['timeout'] = None
                rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
                calls = len(rate_info['calls'])
                if calls > amount:
                    rate_info['timeout'] = now + timeout
                    client.set(key, rate_info)
                    logger.warning('Call {0} is being throttled with a wait of {1}'.format(key, timeout))
                    raise HttpTooManyRequestsException(error='rate_limit_reached',
                                                       error_description='Rate limit reached ({0} in last {1}s)'.format(calls, per))
                client.set(key, rate_info)
            return f(*args, **kwargs)

    def __init__(self, object_type=None, guid=None):
        """
        Initializes a descriptor for a given type. Optionally already providing a guid for the
        instanciator
        """

        # Initialize super class
        super(Descriptor, self).__init__()

        if object_type is None:
            self.initialized = False
        else:
            self.initialized = True

            key = 'ovs_descriptor_{0}'.format(re.sub('[\W_]+', '', str(object_type)))
            self._volatile = VolatileFactory.get_client()
            self._descriptor = self._volatile.get(key)
            if self._descriptor is None:
                Toolbox.log_cache_hit('descriptor', False)
                filename = inspect.getfile(object_type).replace('.pyc', '.py')
                name = filename.replace(os.path.dirname(filename) + os.path.sep, '').replace('.py', '')
                source = os.path.relpath(filename, os.path.dirname(__file__))
                self._descriptor = {'name': name,
                                    'source': source,
                                    'type': object_type.__name__,
                                    'identifier': name + '_' + hashlib.sha256(name + source + object_type.__name__).hexdigest()}
                self._volatile.set(key, self._descriptor)
            else:
                Toolbox.log_cache_hit('descriptor', True)
            self._descriptor['guid'] = guid

 def tearDownClass(cls):
     """
     Clean up the unittest
     """
     fakesleep.monkey_restore()
     cls.volatile = VolatileFactory.get_client()
     cls.volatile.clean()
     cls.persistent = PersistentFactory.get_client()
     cls.persistent.clean()

 def __init__(self, name, wait=None):
     """
     Creates a volatile mutex object
     """
     self._volatile = VolatileFactory.get_client()
     self.name = name
     self._has_lock = False
     self._start = 0
     self._wait = wait

 def setUpClass(cls):
     """
     Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
     This makes sure the unittests can be executed without those libraries installed
     """
     cls.persistent = PersistentFactory.get_client()
     cls.persistent.clean()
     cls.volatile = VolatileFactory.get_client()
     cls.volatile.clean()

 def __init__(self, name, wait=None):
     """
     Creates a volatile mutex object
     """
     self._logger = LogHandler.get('extensions', 'volatile mutex')
     self._volatile = VolatileFactory.get_client()
     self.name = name
     self._has_lock = False
     self._start = 0
     self._wait = wait

    def tearDownClass(cls):
        """
        Tear down changes made during setUpClass
        """
        Configuration._unittest_data = {}

        cls.persistent = PersistentFactory.get_client()
        cls.persistent.clean()

        cls.volatile = VolatileFactory.get_client()
        cls.volatile.clean()

 def log_cache_hit(cache_type, hit):
     """
     Registers a cache hit or miss with a specific type
     """
     volatile = VolatileFactory.get_client()
     key = 'ovs_stats_cache_{0}_{1}'.format(cache_type, 'hit' if hit else 'miss')
     try:
         successfull = volatile.incr(key)
         if not successfull:
             volatile.set(key, 1)
     except:
         pass