Python utils.BucketLister类代码示例

 def list_objects(self, s3_path, dir_op):
     """
     This function yields the appropriate object or objects under a
     common prefix depending if the operation is on objects under a
     common prefix.  It yields the file's source path, size, and last
     update.
     """
     # Short circuit path: if we are not recursing into the s3
     # bucket and a specific path was given, we can just yield
     # that path and not have to call any operation in s3.
     bucket, prefix = find_bucket_key(s3_path)
     if not dir_op and prefix:
         yield self._list_single_object(s3_path)
     else:
         lister = BucketLister(self._client)
         for key in lister.list_objects(bucket=bucket, prefix=prefix,
                                        page_size=self.page_size):
             source_path, response_data = key
             if response_data['Size'] == 0 and source_path.endswith('/'):
                 if self.operation_name == 'delete':
                     # This is to filter out manually created folders
                     # in S3.  They have a size zero and would be
                     # undesirably downloaded.  Local directories
                     # are automatically created when they do not
                     # exist locally.  But user should be able to
                     # delete them.
                     yield source_path, response_data
             elif not dir_op and s3_path != source_path:
                 pass
             else:
                 yield source_path, response_data

 def test_urlencoded_with_unicode_keys(self):
     now = mock.sentinel.now
     self.operation.paginate.return_value = [
         (None, {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': '%E2%9C%93', 'Size': 1}]}),
     ]
     lister = BucketLister(self.operation, self.endpoint, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     # And note how it's been converted to '\r'.
     self.assertEqual(objects, [(u'foo/\u2713', 1, now)])

 def test_urlencoded_with_unicode_keys(self):
     now = mock.sentinel.now
     self.client.get_paginator.return_value.paginate = self.fake_paginate
     self.responses = [
         {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': '%E2%9C%93', 'Size': 1}]},
     ]
     lister = BucketLister(self.client, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     # And note how it's been converted to '\r'.
     self.assertEqual(objects, [(u'foo/\u2713', 1, now)])

 def test_urlencoded_keys(self):
     # In order to workaround control chars being in key names,
     # we force the urlencoding of the key names and we decode
     # them before yielding them.  For example, note the %0D
     # in bar.txt:
     now = mock.sentinel.now
     self.operation.paginate.return_value = [
         (None, {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'bar%0D.txt', 'Size': 1}]}),
     ]
     lister = BucketLister(self.operation, self.endpoint, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     # And note how it's been converted to '\r'.
     self.assertEqual(objects, [('foo/bar\r.txt', 1, now)])

 def test_list_objects(self):
     now = mock.sentinel.now
     self.operation.paginate.return_value = [
         (None, {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'a', 'Size': 1},
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'b', 'Size': 2},]}),
         (None, {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'c', 'Size': 3},
         ]}),
     ]
     lister = BucketLister(self.operation, self.endpoint, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
                                ('foo/c', 3, now)])

 def test_list_objects_passes_in_extra_args(self):
     self.client.get_paginator.return_value.paginate.return_value = [
         {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'mykey', 'Size': 3}
         ]}
     ]
     lister = BucketLister(self.client, self.date_parser)
     list(
         lister.list_objects(
             bucket='mybucket', extra_args={'RequestPayer': 'requester'}
         )
     )
     self.client.get_paginator.return_value.paginate.assert_called_with(
         Bucket='mybucket', PaginationConfig={'PageSize': None},
         RequestPayer='requester'
     )

 def test_list_objects(self):
     now = mock.sentinel.now
     self.client.get_paginator.return_value.paginate = self.fake_paginate
     self.responses = [
         {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'a', 'Size': 1},
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'b', 'Size': 2}]},
         {'Contents': [
             {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'c', 'Size': 3}]}
     ]
     lister = BucketLister(self.client, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
                                ('foo/c', 3, now)])

 def test_urlencoded_keys(self):
     # In order to workaround control chars being in key names,
     # we force the urlencoding of the key names and we decode
     # them before yielding them.  For example, note the %0D
     # in bar.txt:
     now = mock.sentinel.now
     self.client.get_paginator.return_value.paginate = self.fake_paginate
     individual_response_element = {
         'LastModified': '2014-02-27T04:20:38.000Z',
         'Key': 'bar%0D.txt', 'Size': 1}
     self.responses = [
         {'Contents': [individual_response_element]}
     ]
     lister = BucketLister(self.client, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     # And note how it's been converted to '\r'.
     self.assertEqual(
         objects, [('foo/bar\r.txt', individual_response_element)])
     self.assertEqual(individual_response_element['LastModified'], now)

 def test_list_objects(self):
     now = mock.sentinel.now
     self.client.get_paginator.return_value.paginate = self.fake_paginate
     individual_response_elements = [
         {'LastModified': '2014-02-27T04:20:38.000Z',
          'Key': 'a', 'Size': 1},
         {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'b', 'Size': 2},
         {'LastModified': '2014-02-27T04:20:38.000Z',
              'Key': 'c', 'Size': 3}
     ]
     self.responses = [
         {'Contents': individual_response_elements[0:2]},
         {'Contents': [individual_response_elements[2]]}
     ]
     lister = BucketLister(self.client, self.date_parser)
     objects = list(lister.list_objects(bucket='foo'))
     self.assertEqual(objects,
         [('foo/a', individual_response_elements[0]),
          ('foo/b', individual_response_elements[1]),
          ('foo/c', individual_response_elements[2])])
     for individual_response in individual_response_elements:
         self.assertEqual(individual_response['LastModified'], now)

 def test_list_objects(self):
     now = mock.sentinel.now
     self.client.get_paginator.return_value.paginate = self.fake_paginate
     individual_response_elements = [
         {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "a", "Size": 1},
         {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "b", "Size": 2},
         {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "c", "Size": 3},
     ]
     self.responses = [
         {"Contents": individual_response_elements[0:2]},
         {"Contents": [individual_response_elements[2]]},
     ]
     lister = BucketLister(self.client, self.date_parser)
     objects = list(lister.list_objects(bucket="foo"))
     self.assertEqual(
         objects,
         [
             ("foo/a", individual_response_elements[0]),
             ("foo/b", individual_response_elements[1]),
             ("foo/c", individual_response_elements[2]),
         ],
     )
     for individual_response in individual_response_elements:
         self.assertEqual(individual_response["LastModified"], now)