Python paginate.DeprecatedPaginator类代码示例

class TestPaginatorPageSize(unittest.TestCase):
    def setUp(self):
        self.operation = mock.Mock()
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": ["Users", "Groups"],
            'limit_key': 'MaxKeys',
        }
        self.operation.pagination = self.paginate_config
        self.paginator = Paginator(self.operation, self.paginate_config)
        self.endpoint = mock.Mock()

    def test_no_page_size(self):
        kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        pages = self.paginator.paginate(self.endpoint, **kwargs)
        pages._inject_starting_params(kwargs)
        self.assertEqual(kwargs, ref_kwargs)

    def test_page_size(self):
        kwargs =  {'arg1': 'foo', 'arg2': 'bar', 'page_size': 5}
        extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        # Note that ``MaxKeys`` in ``setUp()`` is the parameter used for
        # the page size for pagination.
        ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5}
        pages = self.paginator.paginate(self.endpoint, **kwargs)
        pages._inject_starting_params(extracted_kwargs)
        self.assertEqual(extracted_kwargs, ref_kwargs)

class TestMultipleInputKeys(unittest.TestCase):
    def setUp(self):
        self.operation = mock.Mock()
        # Probably the most complicated example we'll see:
        # multiple input/output/result keys.
        self.paginate_config = {
            "output_token": ["Marker1", "Marker2"],
            "input_token": ["InMarker1", "InMarker2"],
            "result_key": ["Users", "Groups"],
        }
        self.operation.pagination = self.paginate_config
        self.paginator = Paginator(self.operation, self.paginate_config)

    def test_build_full_result_with_multiple_input_keys(self):
        responses = [
            (None, {"Users": ["User1", "User2"], "Groups": ["Group1"],
                    "Marker1": "m1", "Marker2": "m2"}),
            (None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
                    "Marker1": "m3", "Marker2": "m4"}),
            (None, {"Users": ["User5"], "Groups": ["Group3"], }),
        ]
        self.operation.call.side_effect = responses
        pages = self.paginator.paginate(None, max_items=3)
        complete = pages.build_full_result()
        self.assertEqual(complete,
                         {"Users": ['User1', 'User2', 'User3'],
                          "Groups": ['Group1', 'Group2'],
                          "NextToken": "m1___m2___1"})

    def test_resume_with_multiple_input_keys(self):
        responses = [
            (None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
                    "Marker1": "m3", "Marker2": "m4"}),
            (None, {"Users": ["User5"], "Groups": ["Group3"], }),
        ]
        self.operation.call.side_effect = responses
        pages = self.paginator.paginate(None, max_items=1,
                                        starting_token='m1___m2___1')
        complete = pages.build_full_result()
        self.assertEqual(complete,
                         {"Users": ['User4'],
                          "Groups": [],
                          "NextToken": "m3___m4"})
        self.assertEqual(
            self.operation.call.call_args_list,
            [mock.call(None, InMarker1='m1', InMarker2='m2'),])

    def test_result_key_exposed_on_paginator(self):
        self.assertEqual(
            [rk.expression for rk in self.paginator.result_keys],
            ['Users', 'Groups']
        )

    def test_result_key_exposed_on_page_iterator(self):
        pages = self.paginator.paginate(None, max_items=3)
        self.assertEqual(
            [rk.expression for rk in pages.result_keys],
            ['Users', 'Groups']
        )

 def test_next_token_on_page_boundary(self):
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User1"], "Marker": "m1"}),
         (None, {"Users": ["User2"], "Marker": "m2"}),
         (None, {"Users": ["User3"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         paginator.paginate(None, max_items=2).build_full_result(),
         {'Users': ['User1', 'User2'], 'NextToken': 'm2'})

 def test_max_items_can_be_specified(self):
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User1"], "Marker": "m1"}),
         (None, {"Users": ["User2"], "Marker": "m2"}),
         (None, {"Users": ["User3"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         paginator.paginate(None, max_items=1).build_full_result(),
         {'Users': ['User1'], 'NextToken': 'm1'})

 def test_max_items_exceeds_actual_amount(self):
     # Because MaxItems=10 > number of users (3), we should just return
     # all of the users.
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User1"], "Marker": "m1"}),
         (None, {"Users": ["User2"], "Marker": "m2"}),
         (None, {"Users": ["User3"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         paginator.paginate(None, max_items=10).build_full_result(),
         {'Users': ['User1', 'User2', 'User3']})

 def test_max_items_as_strings(self):
     # Some services (route53) model MaxItems as a string type.
     # We need to be able to handle this case.
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User1"], "Marker": "m1"}),
         (None, {"Users": ["User2"], "Marker": "m2"}),
         (None, {"Users": ["User3"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         # Note max_items is a string here.
         paginator.paginate(None, max_items='1').build_full_result(),
         {'Users': ['User1'], 'NextToken': 'm1'})

 def test_max_items_can_be_specified_truncates_response(self):
     # We're saying we only want 4 items, but notice that the second
     # page of results returns users 4-6 so we have to truncated
     # part of that second page.
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}),
         (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
         (None, {"Users": ["User7"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         paginator.paginate(None, max_items=4).build_full_result(),
         {'Users': ['User1', 'User2', 'User3', 'User4'],
          'NextToken': 'm1___1'})

class TestMultipleTokens(unittest.TestCase):
    def setUp(self):
        self.operation = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            "output_token": ["ListBucketResults.NextKeyMarker",
                             "ListBucketResults.NextUploadIdMarker"],
            "input_token": ["key_marker", "upload_id_marker"],
            "result_key": 'Foo',
        }
        self.operation.pagination = self.paginate_config
        self.paginator = Paginator(self.operation, self.paginate_config)

    def test_s3_list_multipart_uploads(self):
        responses = [
            (None, {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1",
                    "NextUploadIdMarker": "up1"}}),
            (None, {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2",
                    "NextUploadIdMarker": "up2"}}),
            (None, {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3",
                    "NextUploadIdMarker": "up3"}}),
            (None, {}),
        ]
        self.operation.call.side_effect = responses
        list(self.paginator.paginate(None))
        self.assertEqual(
            self.operation.call.call_args_list,
            [mock.call(None),
             mock.call(None, key_marker='key1', upload_id_marker='up1'),
             mock.call(None, key_marker='key2', upload_id_marker='up2'),
             mock.call(None, key_marker='key3', upload_id_marker='up3'),
             ])

 def test_include_with_nested_result_keys(self):
     self.paginate_config['result_key'] = 'Result.Key'
     self.paginate_config['non_aggregate_keys'] = [
         'Outer', 'Result.Inner',
     ]
     self.operation.pagination = self.paginate_config
     self.paginator = Paginator(self.operation, self.paginate_config)
     self.set_responses([
         # The non result keys shows hypothetical
         # example.  This doesn't actually happen,
         # but in the case where the non result keys
         # are different across pages, we use the values
         # from the first page.
         {'Result': {'Key': ['foo'], 'Inner': 'v1'},
          'Outer': 'v2', 'NextToken': 't1'},
         {'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'},
          'Outer': 'v4', 'NextToken': 't2'},
         {'Result': {'Key': ['qux'], 'Inner': 'v5'},
          'Outer': 'v6', 'NextToken': 't3'},
     ])
     pages = self.paginator.paginate(None)
     actual = pages.build_full_result()
     self.assertEqual(pages.non_aggregate_part,
                      {'Outer': 'v2', 'Result': {'Inner': 'v1'}})
     expected = {
         'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'},
         'Outer': 'v2',
     }
     self.assertEqual(actual, expected)

 def test_next_token_with_or_expression(self):
     self.operation.pagination = {
         'output_token': 'NextToken || NextToken2',
         'input_token': 'NextToken',
         'result_key': 'Foo',
     }
     self.paginator = Paginator(self.operation, self.operation.pagination)
     # Verify that despite varying between NextToken and NextToken2
     # we still can extract the right next tokens.
     responses = [
         (None, {'NextToken': 'token1'}),
         (None, {'NextToken2': 'token2'}),
         # The first match found wins, so because NextToken is
         # listed before NextToken2 in the 'output_tokens' config,
         # 'token3' is chosen over 'token4'.
         (None, {'NextToken': 'token3', 'NextToken2': 'token4'}),
         (None, {'not_next_token': 'foo'}),
     ]
     self.operation.call.side_effect = responses
     actual = list(self.paginator.paginate(None))
     self.assertEqual(
         self.operation.call.call_args_list,
         [mock.call(None),
          mock.call(None, NextToken='token1'),
          mock.call(None, NextToken='token2'),
          mock.call(None, NextToken='token3'),])

 def setUp(self):
     self.operation = mock.Mock()
     self.paginate_config = {
         'output_token': 'NextToken',
         'input_token': 'NextToken',
         'result_key': 'Foo',
     }
     self.operation.pagination = self.paginate_config
     self.paginator = Paginator(self.operation, self.paginate_config)

 def test_resume_next_marker_mid_page(self):
     # This is a simulation of picking up from the response
     # from test_max_items_can_be_specified_truncates_response
     # We got the first 4 users, when we pick up we should get
     # User5 - User7.
     paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
         (None, {"Users": ["User7"]}),
     ]
     self.operation.call.side_effect = responses
     self.assertEqual(
         paginator.paginate(None, starting_token='m1___1').build_full_result(),
         {'Users': ['User5', 'User6', 'User7']})
     self.assertEqual(
         self.operation.call.call_args_list,
         [mock.call(None, Marker='m1'),
          mock.call(None, Marker='m2'),])

class TestExpressionKeyIterators(unittest.TestCase):
    def setUp(self):
        self.operation = mock.Mock()
        # This is something like what we'd see in RDS.
        self.paginate_config = {
            "input_token": "Marker",
            "output_token": "Marker",
            "limit_key": "MaxRecords",
            "result_key": "EngineDefaults.Parameters"
        }
        self.operation.pagination = self.paginate_config
        self.paginator = Paginator(self.operation, self.paginate_config)
        self.responses = [
            (None, {
                "EngineDefaults": {"Parameters": ["One", "Two"]
            }, "Marker": "m1"}),
            (None, {
                "EngineDefaults": {"Parameters": ["Three", "Four"]
            }, "Marker": "m2"}),
            (None, {"EngineDefaults": {"Parameters": ["Five"]}}),
        ]

    def test_result_key_iters(self):
        self.operation.call.side_effect = self.responses
        pages = self.paginator.paginate(None)
        iterators = pages.result_key_iters()
        self.assertEqual(len(iterators), 1)
        self.assertEqual(list(iterators[0]),
                         ['One', 'Two', 'Three', 'Four', 'Five'])

    def test_build_full_result_with_single_key(self):
        self.operation.call.side_effect = self.responses
        pages = self.paginator.paginate(None)
        complete = pages.build_full_result()
        self.assertEqual(complete, {
            'EngineDefaults': {
                'Parameters': ['One', 'Two', 'Three', 'Four', 'Five']
            },
        })

class TestPaginatorWithPathExpressions(unittest.TestCase):
    def setUp(self):
        self.operation = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            'output_token': [
                'NextMarker || ListBucketResult.Contents[-1].Key'],
            'input_token': 'next_marker',
            'result_key': 'Contents',
        }
        self.operation.pagination = self.paginate_config
        self.paginator = Paginator(self.operation, self.paginate_config)

    def test_s3_list_objects(self):
        responses = [
            (None, {'NextMarker': 'token1'}),
            (None, {'NextMarker': 'token2'}),
            (None, {'not_next_token': 'foo'})]
        self.operation.call.side_effect = responses
        list(self.paginator.paginate(None))
        self.assertEqual(
            self.operation.call.call_args_list,
            [mock.call(None),
             mock.call(None, next_marker='token1'),
             mock.call(None, next_marker='token2'),])

    def test_s3_list_object_complex(self):
        responses = [
            (None, {'NextMarker': 'token1'}),
            (None, {'ListBucketResult': {
                'Contents': [{"Key": "first"}, {"Key": "Last"}]}}),
            (None, {'not_next_token': 'foo'})]
        self.operation.call.side_effect = responses
        list(self.paginator.paginate(None))
        self.assertEqual(
            self.operation.call.call_args_list,
            [mock.call(None),
             mock.call(None, next_marker='token1'),
             mock.call(None, next_marker='Last'),])

 def test_more_tokens_is_path_expression(self):
     self.paginate_config = {
         'more_results': 'Foo.IsTruncated',
         'output_token': 'NextToken',
         'input_token': 'NextToken',
         'result_key': 'Bar',
     }
     self.operation.pagination = self.paginate_config
     self.paginator = Paginator(self.operation, self.paginate_config)
     responses = [
         (None, {'Foo': {'IsTruncated': True}, 'NextToken': 'token1'}),
         (None, {'Foo': {'IsTruncated': False}, 'NextToken': 'token2'}),
     ]
     self.operation.call.side_effect = responses
     list(self.paginator.paginate(None))
     self.assertEqual(
         self.operation.call.call_args_list,
         [mock.call(None),
          mock.call(None, NextToken='token1'),])