Python ratelimit.filter_factory函数代码示例

    def test_registered_defaults(self):

        def check_key_is_absent(key):
            try:
                swift_info[key]
            except KeyError as err:
                if key not in str(err):
                    raise

        test_limits = {'account_ratelimit': 1,
                       'max_sleep_time_seconds': 60,
                       'container_ratelimit_0': 0,
                       'container_ratelimit_10': 10,
                       'container_ratelimit_50': 50,
                       'container_listing_ratelimit_0': 0,
                       'container_listing_ratelimit_10': 10,
                       'container_listing_ratelimit_50': 50}

        ratelimit.filter_factory(test_limits)('have to pass in an app')
        swift_info = utils.get_swift_info()
        self.assertIn('ratelimit', swift_info)
        self.assertEqual(swift_info['ratelimit']
                         ['account_ratelimit'], 1.0)
        self.assertEqual(swift_info['ratelimit']
                         ['max_sleep_time_seconds'], 60.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][0][0], 0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][0][1], 0.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][1][0], 10)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][1][1], 10.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][2][0], 50)
        self.assertEqual(swift_info['ratelimit']
                         ['container_ratelimits'][2][1], 50.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][0][0], 0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][0][1], 0.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][1][0], 10)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][1][1], 10.0)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][2][0], 50)
        self.assertEqual(swift_info['ratelimit']
                         ['container_listing_ratelimits'][2][1], 50.0)

        # these were left out on purpose
        for key in ['log_sleep_time_seconds', 'clock_accuracy',
                    'rate_buffer_seconds', 'ratelimit_whitelis',
                    'ratelimit_blacklist']:
            check_key_is_absent(key)

    def test_ratelimit_blacklist(self):
        global time_ticker
        current_rate = 2
        conf_dict = {'account_ratelimit': current_rate,
                     'max_sleep_time_seconds': 2,
                     'account_whitelist': 'a',
                     'account_blacklist': 'b'}
        self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
        self.test_ratelimit.logger = FakeLogger()
        self.test_ratelimit.BLACK_LIST_SLEEP = 0
        ratelimit.http_connect = mock_http_connect(204)
        req = Request.blank('/v/b/c')
        req.environ['swift.cache'] = FakeMemcache()

        class rate_caller(Thread):

            def __init__(self, parent):
                Thread.__init__(self)
                self.parent = parent

            def run(self):
                self.result = self.parent.test_ratelimit(req.environ,
                                                         start_response)
        nt = 5
        threads = []
        for i in range(nt):
            rc = rate_caller(self)
            rc.start()
            threads.append(rc)
        for thread in threads:
            thread.join()
        the_497s = [
            t for t in threads if ''.join(t.result).startswith('Your account')]
        self.assertEquals(len(the_497s), 5)
        self.assertEquals(time_ticker, 0)

    def test_ratelimit_max_rate_double_container(self):
        global time_ticker
        global time_override
        current_rate = 2
        conf_dict = {'container_ratelimit_0': current_rate,
                     'clock_accuracy': 100,
                     'max_sleep_time_seconds': 1}
        self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
        self.test_ratelimit.log_sleep_time_seconds = .00001
        req = Request.blank('/v1/a/c/o')
        req.method = 'PUT'
        req.environ['swift.cache'] = FakeMemcache()
        req.environ['swift.cache'].set(
            get_cache_key('a', 'c'),
            {'object_count': 1})

        time_override = [0, 0, 0, 0, None]
        # simulates 4 requests coming in at same time, then sleeping
        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
                        lambda *args, **kwargs: {}):
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], b'Slow down')
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], b'Slow down')
            mock_sleep(.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], b'Some Content')

    def test_ratelimit_max_rate_multiple_acc(self):
        num_calls = 4
        current_rate = 2
        conf_dict = {"account_ratelimit": current_rate, "max_sleep_time_seconds": 2}
        fake_memcache = FakeMemcache()

        the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
        the_app.memcache_client = fake_memcache
        req = lambda: None
        req.method = "PUT"
        req.environ = {}

        class rate_caller(threading.Thread):
            def __init__(self, name):
                self.myname = name
                threading.Thread.__init__(self)

            def run(self):
                for j in range(num_calls):
                    self.result = the_app.handle_ratelimit(req, self.myname, "c", None)

        with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
            nt = 15
            begin = time.time()
            threads = []
            for i in range(nt):
                rc = rate_caller("a%s" % i)
                rc.start()
                threads.append(rc)
            for thread in threads:
                thread.join()

            time_took = time.time() - begin
            self.assertEqual(1.5, round(time_took, 1))

    def test_ratelimit_max_rate_double_container_listing(self):
        global time_ticker
        global time_override
        current_rate = 2
        conf_dict = {"container_listing_ratelimit_0": current_rate, "clock_accuracy": 100, "max_sleep_time_seconds": 1}
        self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
        ratelimit.http_connect = mock_http_connect(204)
        self.test_ratelimit.log_sleep_time_seconds = 0.00001
        req = Request.blank("/v/a/c")
        req.method = "GET"
        req.environ["swift.cache"] = FakeMemcache()
        req.environ["swift.cache"].set(get_cache_key("a", "c"), {"object_count": 1})

        with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
            time_override = [0, 0, 0, 0, None]
            # simulates 4 requests coming in at same time, then sleeping
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "Slow down")
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "Slow down")
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "204 No Content")
            mc = self.test_ratelimit.memcache_client
            try:
                self.test_ratelimit.memcache_client = None
                self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "n", "c", None), None)
            finally:
                self.test_ratelimit.memcache_client = mc

    def test_ratelimit_max_rate_double(self):
        global time_ticker
        global time_override
        current_rate = 2
        conf_dict = {"account_ratelimit": current_rate, "clock_accuracy": 100, "max_sleep_time_seconds": 1}
        self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
        ratelimit.http_connect = mock_http_connect(204)
        self.test_ratelimit.log_sleep_time_seconds = 0.00001
        req = Request.blank("/v/a/c")
        req.method = "PUT"
        req.environ["swift.cache"] = FakeMemcache()

        time_override = [0, 0, 0, 0, None]
        # simulates 4 requests coming in at same time, then sleeping
        with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "Slow down")
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "Slow down")
            mock_sleep(0.1)
            r = self.test_ratelimit(req.environ, start_response)
            self.assertEqual(r[0], "204 No Content")

    def test_get_ratelimitable_key_tuples(self):
        current_rate = 13
        conf_dict = {"account_ratelimit": current_rate, "container_ratelimit_3": 200}
        fake_memcache = FakeMemcache()
        fake_memcache.store[get_cache_key("a", "c")] = {"object_count": "5"}
        the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
        the_app.memcache_client = fake_memcache
        req = lambda: None
        req.environ = {"swift.cache": fake_memcache, "PATH_INFO": "/v1/a/c/o"}
        with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
            req.method = "DELETE"
            self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", None, None)), 0)
            req.method = "PUT"
            self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None)), 1)
            req.method = "DELETE"
            self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None)), 1)
            req.method = "GET"
            self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", "o")), 0)
            req.method = "PUT"
            self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", "o")), 1)

        req.method = "PUT"
        self.assertEqual(len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit=10)), 2)
        self.assertEqual(
            the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit=10)[1],
            ("ratelimit/global-write/a", 10),
        )

        req.method = "PUT"
        self.assertEqual(
            len(the_app.get_ratelimitable_key_tuples(req, "a", "c", None, global_ratelimit="notafloat")), 1
        )

    def test_registered_defaults(self):
        def check_key_is_absnet(key):
            try:
                swift_info[key]
            except KeyError as err:
                if key not in err:
                    raise

        test_limits = {
            "account_ratelimit": 1,
            "max_sleep_time_seconds": 60,
            "container_ratelimit_0": 0,
            "container_ratelimit_10": 10,
            "container_ratelimit_50": 50,
            "container_listing_ratelimit_0": 0,
            "container_listing_ratelimit_10": 10,
            "container_listing_ratelimit_50": 50,
        }

        ratelimit.filter_factory(test_limits)("have to pass in an app")
        swift_info = utils.get_swift_info()
        self.assertTrue("ratelimit" in swift_info)
        self.assertEqual(swift_info["ratelimit"]["account_ratelimit"], 1.0)
        self.assertEqual(swift_info["ratelimit"]["max_sleep_time_seconds"], 60.0)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][0][0], 0)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][0][1], 0.0)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][1][0], 10)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][1][1], 10.0)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][2][0], 50)
        self.assertEqual(swift_info["ratelimit"]["container_ratelimits"][2][1], 50.0)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][0][0], 0)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][0][1], 0.0)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][1][0], 10)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][1][1], 10.0)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][2][0], 50)
        self.assertEqual(swift_info["ratelimit"]["container_listing_ratelimits"][2][1], 50.0)

        # these were left out on purpose
        for key in [
            "log_sleep_time_seconds",
            "clock_accuracy",
            "rate_buffer_seconds",
            "ratelimit_whitelis",
            "ratelimit_blacklist",
        ]:
            check_key_is_absnet(key)

 def test_get_maxrate(self):
     conf_dict = {"container_ratelimit_10": 200, "container_ratelimit_50": 100, "container_ratelimit_75": 30}
     test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     test_ratelimit.logger = FakeLogger()
     self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 0), None)
     self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 5), None)
     self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 10), 200)
     self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 60), 72)
     self.assertEqual(ratelimit.get_maxrate(test_ratelimit.container_ratelimits, 160), 30)

 def test_no_memcache(self):
     current_rate = 13
     num_calls = 5
     conf_dict = {'account_ratelimit': current_rate}
     self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     ratelimit.http_connect = mock_http_connect(204)
     req = Request.blank('/v/a')
     req.environ['swift.cache'] = None
     make_app_call = lambda: self.test_ratelimit(req.environ,
                                                 start_response)
     self._run(make_app_call, num_calls, current_rate)

 def test_no_memcache(self):
     current_rate = 13
     num_calls = 5
     conf_dict = {'account_ratelimit': current_rate}
     self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     req = Request.blank('/v1/a')
     req.environ['swift.cache'] = None
     make_app_call = lambda: self.test_ratelimit(req.environ,
                                                 start_response)
     begin = time.time()
     self._run(make_app_call, num_calls, current_rate, check_time=False)
     time_took = time.time() - begin
     self.assertEqual(round(time_took, 1), 0)  # no memcache, no limiting

 def test_ratelimit_old_memcache_format(self):
     current_rate = 13
     conf_dict = {"account_ratelimit": current_rate, "container_ratelimit_3": 200}
     fake_memcache = FakeMemcache()
     fake_memcache.store[get_cache_key("a", "c")] = {"container_size": 5}
     the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
     the_app.memcache_client = fake_memcache
     req = lambda: None
     req.method = "PUT"
     req.environ = {"PATH_INFO": "/v1/a/c/o", "swift.cache": fake_memcache}
     with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
         tuples = the_app.get_ratelimitable_key_tuples(req, "a", "c", "o")
         self.assertEqual(tuples, [("ratelimit/a/c", 200.0)])

 def test_ratelimit_old_white_black_list(self):
     global time_ticker
     current_rate = 2
     conf_dict = {
         "account_ratelimit": current_rate,
         "max_sleep_time_seconds": 2,
         "account_whitelist": "a",
         "account_blacklist": "b",
     }
     self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     req = Request.blank("/")
     with mock.patch.object(self.test_ratelimit, "memcache_client", FakeMemcache()):
         self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "a", "c", "o"), None)
         self.assertEqual(self.test_ratelimit.handle_ratelimit(req, "b", "c", "o").status_int, 497)

 def test_ratelimit_set_incr(self):
     current_rate = 5
     num_calls = 50
     conf_dict = {"account_ratelimit": current_rate}
     self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     ratelimit.http_connect = mock_http_connect(204)
     req = Request.blank("/v/a/c")
     req.method = "PUT"
     req.environ["swift.cache"] = FakeMemcache()
     req.environ["swift.cache"].init_incr_return_neg = True
     make_app_call = lambda: self.test_ratelimit(req.environ, start_response)
     begin = time.time()
     with mock.patch("swift.common.middleware.ratelimit.get_account_info", lambda *args, **kwargs: {}):
         self._run(make_app_call, num_calls, current_rate, check_time=False)
         self.assertEqual(round(time.time() - begin, 1), 9.8)

 def test_ratelimit_set_incr(self):
     current_rate = 5
     num_calls = 50
     conf_dict = {'account_ratelimit': current_rate}
     self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
     ratelimit.http_connect = mock_http_connect(204)
     req = Request.blank('/v/a/c')
     req.method = 'PUT'
     req.environ['swift.cache'] = FakeMemcache()
     req.environ['swift.cache'].init_incr_return_neg = True
     make_app_call = lambda: self.test_ratelimit(req.environ,
                                                 start_response)
     begin = time.time()
     self._run(make_app_call, num_calls, current_rate, check_time=False)
     self.assertEquals(round(time.time() - begin, 1), 9.8)