Python base.get_container_memcache_key函数代码示例

    def GETorHEAD(self, req):
        """Handler for HTTP GET/HEAD requests."""
        if not self.account_info(self.account_name)[1]:
            return HTTPNotFound(request=req)
        part, nodes = self.app.container_ring.get_nodes(
                        self.account_name, self.container_name)
        shuffle(nodes)
        resp = self.GETorHEAD_base(req, _('Container'), part, nodes,
                req.path_info, len(nodes))

        if self.app.memcache:
            # set the memcache container size for ratelimiting
            cache_key = get_container_memcache_key(self.account_name,
                                                   self.container_name)
            self.app.memcache.set(cache_key,
              {'status': resp.status_int,
               'read_acl': resp.headers.get('x-container-read'),
               'write_acl': resp.headers.get('x-container-write'),
               'sync_key': resp.headers.get('x-container-sync-key'),
               'count': resp.headers.get('x-container-object-count'),
               'bytes': resp.headers.get('x-container-bytes-used'),
               'versions': resp.headers.get('x-versions-location')},
                                  timeout=self.app.recheck_container_existence)

        if 'swift.authorize' in req.environ:
            req.acl = resp.headers.get('x-container-read')
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not req.environ.get('swift_owner', False):
            for key in ('x-container-read', 'x-container-write',
                        'x-container-sync-key', 'x-container-sync-to'):
                if key in resp.headers:
                    del resp.headers[key]
        return resp

    def test_ratelimit_max_rate_double_container(self):
        global time_ticker
        global time_override
        current_rate = 2
        conf_dict = {'container_ratelimit_0': current_rate,
                     'clock_accuracy': 100,
                     'max_sleep_time_seconds': 1}
        self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
        ratelimit.http_connect = mock_http_connect(204)
        self.test_ratelimit.log_sleep_time_seconds = .00001
        req = Request.blank('/v/a/c/o')
        req.method = 'PUT'
        req.environ['swift.cache'] = FakeMemcache()
        req.environ['swift.cache'].set(
            get_container_memcache_key('a', 'c'),
            {'container_size': 1})

        time_override = [0, 0, 0, 0, None]
        # simulates 4 requests coming in at same time, then sleeping
        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                        lambda *args, **kwargs: {}):
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEquals(r[0], 'Slow down')
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEquals(r[0], 'Slow down')
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEquals(r[0], '204 No Content')

    def get_ratelimitable_key_tuples(self, req_method, account_name,
                                     container_name=None, obj_name=None):
        """
        Returns a list of key (used in memcache), ratelimit tuples. Keys
        should be checked in order.

        :param req_method: HTTP method
        :param account_name: account name from path
        :param container_name: container name from path
        :param obj_name: object name from path
        """
        keys = []
        # COPYs are not limited
        if self.account_ratelimit and \
                account_name and container_name and not obj_name and \
                req_method in ('PUT', 'DELETE'):
            keys.append(("ratelimit/%s" % account_name,
                         self.account_ratelimit))

        if account_name and container_name and obj_name and \
                req_method in ('PUT', 'DELETE', 'POST'):
            container_size = None
            memcache_key = get_container_memcache_key(account_name,
                                                      container_name)
            container_info = self.memcache_client.get(memcache_key)
            if isinstance(container_info, dict):
                container_size = container_info.get(
                    'object_count', container_info.get('container_size', 0))
                container_rate = self.get_container_maxrate(container_size)
                if container_rate:
                    keys.append(("ratelimit/%s/%s" % (account_name,
                                                      container_name),
                                 container_rate))
        return keys

    def GETorHEAD(self, req):
        """Handler for HTTP GET/HEAD requests."""
        if not self.account_info(self.account_name, req)[1]:
            return HTTPNotFound(request=req)
        part = self.app.container_ring.get_part(
            self.account_name, self.container_name)
        resp = self.GETorHEAD_base(
            req, _('Container'), self.app.container_ring, part, req.path_info)
        if self.app.memcache:
            # set the memcache container size for ratelimiting
            cache_key = get_container_memcache_key(self.account_name,
                                                   self.container_name)
            self.app.memcache.set(
                cache_key,
                headers_to_container_info(resp.headers, resp.status_int),
                time=self.app.recheck_container_existence)

        if 'swift.authorize' in req.environ:
            req.acl = resp.headers.get('x-container-read')
            aresp = req.environ['swift.authorize'](req)
            if aresp:
                return aresp
        if not req.environ.get('swift_owner', False):
            for key in ('x-container-read', 'x-container-write',
                        'x-container-sync-key', 'x-container-sync-to'):
                if key in resp.headers:
                    del resp.headers[key]
        return resp

 def PUT(self, req):
     """HTTP PUT request handler."""
     error_response = \
         self.clean_acls(req) or check_metadata(req, 'container')
     if error_response:
         return error_response
     if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
         resp = HTTPBadRequest(request=req)
         resp.body = 'Container name length of %d longer than %d' % \
                     (len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
         return resp
     account_partition, accounts, container_count = \
         self.account_info(self.account_name,
                           autocreate=self.app.account_autocreate)
     if self.app.max_containers_per_account > 0 and \
             container_count >= self.app.max_containers_per_account and \
             self.account_name not in self.app.max_containers_whitelist:
         resp = HTTPForbidden(request=req)
         resp.body = 'Reached container limit of %s' % \
                     self.app.max_containers_per_account
         return resp
     if not accounts:
         return HTTPNotFound(request=req)
     container_partition, containers = self.app.container_ring.get_nodes(
         self.account_name, self.container_name)
     headers = self._backend_requests(req, len(containers),
                                      account_partition, accounts)
     if self.app.memcache:
         cache_key = get_container_memcache_key(self.account_name,
                                                self.container_name)
         self.app.memcache.delete(cache_key)
     resp = self.make_requests(
         req, self.app.container_ring,
         container_partition, 'PUT', req.path_info, headers)
     return resp

    def handle_delete(self, env, start_response, version, account, container,
                      obj):
        """ Handle delete request. """
        memcache_client = cache_from_env(env)
        if not memcache_client:
            return self.app(env, start_response)

        res = [None, None, None]
        result_code = None

        def _start_response(response_status, response_headers, exc_info=None):
            res[0] = response_status
            res[1] = response_headers
            res[2] = exc_info

        resp = self.app(env, _start_response)
        result_code = self._get_status_int(res[0])
        try:
            if is_success(result_code):
                if obj:
                    memcache_client.delete(
                        get_container_memcache_key(account, container))
                else:
                    memcache_client.delete(get_account_memcache_key(account))
        except Exception, err:
            self.logger.error(
                'Error in [Quota] delete cache: %s' % (err.message))

    def handle_quota_object(self, env, start_response, version, account,
                            container, obj):
        """ Handle quota of container usage and object count. """
        memcache_client = cache_from_env(env)
        object_size = int(env.get('CONTENT_LENGTH') or 0)
        container_count, quota_level = self._get_account_meta(
            env, version, account, memcache_client)
        try:
            container_usage_quota = self.container_usage[quota_level]
            object_count_quota = self.object_count[quota_level]
        except Exception:
            self.logger.warn('Invalid quota_leve %s/%s quota_level[%s].' % (
                account, container, quota_level))
            container_usage_quota = None
            object_count_quota = None
        container_usage, object_count = self._get_container_meta(
            env, version, account, container, memcache_client)
        if container_usage_quota and container_usage >= container_usage_quota:
            self.logger.notice("Container usage over quota, "
                               "request[PUT %s/%s/%s], container_usage[%s] "
                               "object_size[%s] quota[%s]" % (
                                   account, container, obj, container_usage,
                                   object_size, container_usage_quota))
            return HTTPForbidden(body="The usage of container is over quota")(
                env, start_response)
        elif container_usage_quota and (container_usage + object_size >
                                        container_usage_quota):
            self.logger.notice("Container usage over quota, "
                               "request[PUT %s/%s/%s], container_usage[%s] "
                               "object_size[%s] quota[%s]" % (
                                   account, container, obj, container_usage,
                                   object_size, container_usage_quota))
            return HTTPForbidden(body="The usage of container is over quota")(
                env, start_response)
        elif object_count_quota and object_count + 1 > object_count_quota:
            self.logger.notice("Object count over quota, request[PUT %s/%s/%s],"
                               "object_count[%s] quota[%s]" % (
                                   account, container, obj, object_count + 1,
                                   object_count_quota))
            return HTTPForbidden(body="The count of object is over quota")(
                env, start_response)
        elif self.precise_mode and memcache_client:
            res = [None, None, None]
            result_code = None

            def _start_response(response_status, response_headers,
                                exc_info=None):
                res[0] = response_status
                res[1] = response_headers
                res[2] = exc_info

            resp = self.app(env, _start_response)
            result_code = self._get_status_int(res[0])
            if is_success(result_code):
                memcache_client.delete(
                    get_container_memcache_key(account, container))
            start_response(res[0], res[1], res[2])
            return resp
        else:
            return self.app(env, start_response)

 def DELETE(self, req):
     """HTTP DELETE request handler."""
     account_partition, accounts, container_count = \
         self.account_info(self.account_name)
     if not accounts:
         return HTTPNotFound(request=req)
     container_partition, containers = self.app.container_ring.get_nodes(
         self.account_name, self.container_name)
     headers = []
     for account in accounts:
         headers.append({'X-Timestamp': normalize_timestamp(time.time()),
                        'X-Trans-Id': self.trans_id,
                        'X-Account-Host': '%(ip)s:%(port)s' % account,
                        'X-Account-Partition': account_partition,
                        'X-Account-Device': account['device'],
                        'Connection': 'close'})
     if self.app.memcache:
         cache_key = get_container_memcache_key(self.account_name,
                                                self.container_name)
         self.app.memcache.delete(cache_key)
     resp = self.make_requests(req, self.app.container_ring,
                 container_partition, 'DELETE', req.path_info, headers)
     # Indicates no server had the container
     if resp.status_int == HTTP_ACCEPTED:
         return HTTPNotFound(request=req)
     return resp

 def POST(self, req):
     """HTTP POST request handler."""
     error_response = \
         self.clean_acls(req) or check_metadata(req, 'container')
     if error_response:
         return error_response
     account_partition, accounts, container_count = \
         self.account_info(self.account_name,
                           autocreate=self.app.account_autocreate)
     if not accounts:
         return HTTPNotFound(request=req)
     container_partition, containers = self.app.container_ring.get_nodes(
         self.account_name, self.container_name)
     headers = {'X-Timestamp': normalize_timestamp(time.time()),
                'x-trans-id': self.trans_id,
                'Connection': 'close'}
     self.transfer_headers(req.headers, headers)
     if self.app.memcache:
         cache_key = get_container_memcache_key(self.account_name,
                                                self.container_name)
         self.app.memcache.delete(cache_key)
     resp = self.make_requests(req, self.app.container_ring,
             container_partition, 'POST', req.path_info,
             [headers] * len(containers))
     return resp

 def test_get_container_info_env(self):
     cache_key = get_container_memcache_key("account", "cont")
     env_key = 'swift.%s' % cache_key
     req = Request.blank("/v1/account/cont",
                         environ={env_key: {'bytes': 3867},
                                  'swift.cache': FakeCache({})})
     resp = get_container_info(req.environ, 'xxx')
     self.assertEquals(resp['bytes'], 3867)

 def get_container_size(self, account_name, container_name):
     rv = 0
     memcache_key = get_container_memcache_key(account_name,
                                               container_name)
     container_info = self.memcache_client.get(memcache_key)
     if isinstance(container_info, dict):
         rv = container_info.get(
             'object_count', container_info.get('container_size', 0))
     return rv

 def test_ratelimit_old_memcache_format(self):
     current_rate = 13
     conf_dict = {'account_ratelimit': current_rate,
                  'container_ratelimit_3': 200}
     fake_memcache = FakeMemcache()
     fake_memcache.store[get_container_memcache_key('a', 'c')] = \
         {'container_size': 5}
     the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
                                             logger=FakeLogger())
     the_app.memcache_client = fake_memcache
     tuples = the_app.get_ratelimitable_key_tuples('PUT', 'a', 'c', 'o')
     self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])

    def test_get_ratelimitable_key_tuples(self):
        current_rate = 13
        conf_dict = {'account_ratelimit': current_rate,
                     'container_ratelimit_3': 200}
        fake_memcache = FakeMemcache()
        fake_memcache.store[get_container_memcache_key('a', 'c')] = \
            {'object_count': '5'}
        the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
                                                logger=FakeLogger())
        the_app.memcache_client = fake_memcache
        req = lambda: None
        req.environ = {}
        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                        lambda *args, **kwargs: {}):
            req.method = 'DELETE'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', None, None)), 0)
            req.method = 'PUT'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', None)), 1)
            req.method = 'DELETE'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', None)), 1)
            req.method = 'GET'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', 'o')), 0)
            req.method = 'PUT'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', 'o')), 1)

        def get_fake_ratelimit(*args, **kwargs):
            return {'sysmeta': {'global-write-ratelimit': 10}}

        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                        get_fake_ratelimit):
            req.method = 'PUT'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', None)), 2)
            self.assertEquals(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10))

        def get_fake_ratelimit(*args, **kwargs):
            return {'sysmeta': {'global-write-ratelimit': 'notafloat'}}

        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                        get_fake_ratelimit):
            req.method = 'PUT'
            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
                req, 'a', 'c', None)), 1)

 def test_ratelimit_old_memcache_format(self):
     current_rate = 13
     conf_dict = {'account_ratelimit': current_rate,
                  'container_ratelimit_3': 200}
     fake_memcache = FakeMemcache()
     fake_memcache.store[get_container_memcache_key('a', 'c')] = \
         {'container_size': 5}
     the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
     the_app.memcache_client = fake_memcache
     req = lambda: None
     req.method = 'PUT'
     req.environ = {'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache}
     with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                     lambda *args, **kwargs: {}):
         tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
         self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])

 def PUT(self, req):
     """HTTP PUT request handler."""
     start_time = time.time()
     error_response = \
         self.clean_acls(req) or check_metadata(req, 'container')
     if error_response:
         self.app.logger.increment('errors')
         return error_response
     if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
         resp = HTTPBadRequest(request=req)
         resp.body = 'Container name length of %d longer than %d' % \
                     (len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
         self.app.logger.increment('errors')
         return resp
     account_partition, accounts, container_count = \
         self.account_info(self.account_name,
                           autocreate=self.app.account_autocreate)
     if self.app.max_containers_per_account > 0 and \
             container_count >= self.app.max_containers_per_account and \
             self.account_name not in self.app.max_containers_whitelist:
         resp = HTTPForbidden(request=req)
         resp.body = 'Reached container limit of %s' % \
                     self.app.max_containers_per_account
         return resp
     if not accounts:
         self.app.logger.timing_since('PUT.timing', start_time)
         return HTTPNotFound(request=req)
     container_partition, containers = self.app.container_ring.get_nodes(
         self.account_name, self.container_name)
     headers = []
     for account in accounts:
         nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
                     'x-trans-id': self.trans_id,
                     'X-Account-Host': '%(ip)s:%(port)s' % account,
                     'X-Account-Partition': account_partition,
                     'X-Account-Device': account['device'],
                     'Connection': 'close'}
         self.transfer_headers(req.headers, nheaders)
         headers.append(nheaders)
     if self.app.memcache:
         cache_key = get_container_memcache_key(self.account_name,
                                                self.container_name)
         self.app.memcache.delete(cache_key)
     resp = self.make_requests(req, self.app.container_ring,
             container_partition, 'PUT', req.path_info, headers)
     self.app.logger.timing_since('PUT.timing', start_time)
     return resp