Python SwiftService._upload_object_job方法代码示例

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_identical_etag(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'content-length': 30,
                'etag': md5(b'a' * 30).hexdigest()}
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': True,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0})

            self.assertTrue(r['success'])
            self.assertIn('status', r)
            self.assertEqual(r['status'], 'skipped-identical')
            self.assertEqual(mock_conn.put_object.call_count, 0)
            self.assertEqual(mock_conn.head_object.call_count, 1)
            mock_conn.head_object.assert_called_with('test_c', 'test_o')

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_file(self):
        # Uploading a file results in the file object being wrapped in a
        # LengthWrapper. This test sets the options in such a way that much
        # of _upload_object_job is skipped bringing the critical path down
        # to around 60 lines to ease testing.
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            expected_r = {
                'action': 'upload_object',
                'attempts': 2,
                'container': 'test_c',
                'headers': {},
                'large_object': False,
                'object': 'test_o',
                'response_dict': {},
                'status': 'uploaded',
                'success': True,
            }
            expected_mtime = float(os.path.getmtime(f.name))

            mock_conn = mock.Mock()
            mock_conn.put_object.return_value = ''
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            mtime = float(r['headers']['x-object-meta-mtime'])
            self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
            del r['headers']['x-object-meta-mtime']

            self.assertEqual(r['path'], f.name)
            del r['path']

            self._assertDictEqual(r, expected_r)
            self.assertEqual(mock_conn.put_object.call_count, 1)
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=30,
                                                    headers={},
                                                    response_dict={})
            contents = mock_conn.put_object.call_args[0][2]
            self.assertIsInstance(contents, utils.LengthWrapper)
            self.assertEqual(len(contents), 30)
            # This read forces the LengthWrapper to calculate the md5
            # for the read content. This also checks that LengthWrapper was
            # initialized with md5=True
            self.assertEqual(contents.read(), b'a' * 30)
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_stream(self):
        # Streams are wrapped as ReadableToIterable
        with tempfile.TemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            f.seek(0)
            expected_r = {
                'action': 'upload_object',
                'attempts': 2,
                'container': 'test_c',
                'headers': {},
                'large_object': False,
                'object': 'test_o',
                'response_dict': {},
                'status': 'uploaded',
                'success': True,
                'path': None,
            }
            expected_mtime = float(time.time())

            mock_conn = mock.Mock()
            mock_conn.put_object.return_value = ''
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            mtime = float(r['headers']['x-object-meta-mtime'])
            self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
            del r['headers']['x-object-meta-mtime']

            self._assertDictEqual(r, expected_r)
            self.assertEqual(mock_conn.put_object.call_count, 1)
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=None,
                                                    headers={},
                                                    response_dict={})
            contents = mock_conn.put_object.call_args[0][2]
            self.assertIsInstance(contents, utils.ReadableToIterable)
            self.assertEqual(contents.chunk_size, 65536)
            # next retrieves the first chunk of the stream or len(chunk_size)
            # or less, it also forces the md5 to be calculated.
            self.assertEqual(next(contents), b'a' * 30)
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_identical_dlo(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            segment_etag = md5(b'a' * 10).hexdigest()

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'x-object-manifest': 'test_c_segments/test_o/prefix',
                'content-length': 30,
                'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()}
            mock_conn.get_container.side_effect = [
                (None, [{"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/00"},
                        {"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/01"}]),
                (None, [{"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/02"}]),
                (None, {})]
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            with mock.patch('swiftclient.service.get_conn',
                            return_value=mock_conn):
                r = s._upload_object_job(conn=mock_conn,
                                         container='test_c',
                                         source=f.name,
                                         obj='test_o',
                                         options={'changed': False,
                                                  'skip_identical': True,
                                                  'leave_segments': True,
                                                  'header': '',
                                                  'segment_size': 10})

            self.assertIsNone(r.get('error'))
            self.assertTrue(r['success'])
            self.assertEqual('skipped-identical', r.get('status'))
            self.assertEqual(0, mock_conn.put_object.call_count)
            self.assertEqual(1, mock_conn.head_object.call_count)
            self.assertEqual(3, mock_conn.get_container.call_count)
            mock_conn.head_object.assert_called_with('test_c', 'test_o')
            expected = [
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker='', delimiter=None),
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker="test_o/prefix/01", delimiter=None),
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker="test_o/prefix/02", delimiter=None),
            ]
            mock_conn.get_container.assert_has_calls(expected)

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_identical_slo_with_nesting(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            seg_etag = md5(b'a' * 10).hexdigest()
            submanifest = "[%s]" % ",".join(
                ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2)
            submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest()
            manifest = "[%s]" % ",".join([
                '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",'
                '"bytes":20,"hash":"%s"}' % submanifest_etag,
                '{"bytes":10,"hash":"%s"}' % seg_etag])

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'x-static-large-object': True,
                'content-length': 30,
                'etag': md5(submanifest_etag.encode('ascii') +
                            seg_etag.encode('ascii')).hexdigest()}
            mock_conn.get_object.side_effect = [
                ({}, manifest.encode('ascii')),
                ({}, submanifest.encode('ascii'))]
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': True,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 10})

            self.assertIsNone(r.get('error'))
            self.assertTrue(r['success'])
            self.assertEqual('skipped-identical', r.get('status'))
            self.assertEqual(0, mock_conn.put_object.call_count)
            self.assertEqual([mock.call('test_c', 'test_o')],
                             mock_conn.head_object.mock_calls)
            self.assertEqual([
                mock.call('test_c', 'test_o',
                          query_string='multipart-manifest=get'),
                mock.call('test_c_segments', 'test_sub_slo',
                          query_string='multipart-manifest=get'),
            ], mock_conn.get_object.mock_calls)

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import _upload_object_job [as 别名]
    def test_upload_object_job_etag_mismatch(self):
        # The etag test for both streams and files use the same code
        # so only one test should be needed.
        def _consuming_conn(*a, **kw):
            contents = a[2]
            contents.read()  # Force md5 calculation
            return 'badresponseetag'

        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()

            mock_conn = mock.Mock()
            mock_conn.put_object.side_effect = _consuming_conn
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            self.assertEqual(r['success'], False)
            self.assertIn('error', r)
            self.assertIn('md5 mismatch', str(r['error']))

            self.assertEqual(mock_conn.put_object.call_count, 1)
            expected_headers = {'x-object-meta-mtime': mock.ANY}
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=30,
                                                    headers=expected_headers,
                                                    response_dict={})

            contents = mock_conn.put_object.call_args[0][2]
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())