Python service.SwiftService类代码示例

def saveObjsBackend(objs, backend, config):

    if(backend == 'hdfs'):
        for obj in objs:
            try:
                # obj[0] is hdfs path and obj[1] is local filesystem path
                if '.py' in obj[1]:  # If you are uploading a module do not allow it to be overwritten
                    subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]])
                else:  # If it is the metadata then it has to be overwritten everytime
                    subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]])
            except Exception as e:
                shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
                raise RuntimeError(e)

    elif(backend == 'swift'):
        options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os  .environ['OS_TENANT_NAME']}
        swiftService = SwiftService(options=options)
        objects = []
        for obj in objs:
            objects.append(SwiftUploadObject(obj[1], object_name=obj[0]))

            swiftUpload = swiftService.upload(container='containerModules', objects=objects)
            for uploaded in swiftUpload:
                if("error" in uploaded.keys()):
                    shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
                    raise RuntimeError(uploaded['error'])

    elif(backend == 'nfs'):
        for obj in objs:
            shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0])

    print('Metadata/Module changed and uploaded')

    def test_upload_object_job_identical_etag(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'content-length': 30,
                'etag': md5(b'a' * 30).hexdigest()}
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': True,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0})

            self.assertTrue(r['success'])
            self.assertIn('status', r)
            self.assertEqual(r['status'], 'skipped-identical')
            self.assertEqual(mock_conn.put_object.call_count, 0)
            self.assertEqual(mock_conn.head_object.call_count, 1)
            mock_conn.head_object.assert_called_with('test_c', 'test_o')

    def test_delete_object_dlo_support(self):
        mock_q = Queue()
        s = SwiftService()
        mock_conn = self._get_mock_connection()
        expected_r = self._get_expected({
            'action': 'delete_object',
            'success': True,
            'dlo_segments_deleted': True
        })
        # A DLO object is determined in _delete_object by heading the object
        # and checking for the existence of a x-object-manifest header.
        # Mock that here.
        mock_conn.head_object = Mock(
            return_value={'x-object-manifest': 'manifest_c/manifest_p'}
        )
        mock_conn.get_container = Mock(
            side_effect=[(None, [{'name': 'test_seg_1'},
                                 {'name': 'test_seg_2'}]),
                         (None, {})]
        )

        def get_mock_list_conn(options):
            return mock_conn

        with mock.patch('swiftclient.service.get_conn', get_mock_list_conn):
            r = s._delete_object(
                mock_conn, 'test_c', 'test_o', self.opts, mock_q
            )

        self._assertDictEqual(expected_r, r)
        expected = [
            mock.call('test_c', 'test_o', query_string=None, response_dict={}),
            mock.call('manifest_c', 'test_seg_1', response_dict={}),
            mock.call('manifest_c', 'test_seg_2', response_dict={})]
        mock_conn.delete_object.assert_has_calls(expected, any_order=True)

class SwiftClient(object):
    """Client for Swift object/blob store of Openstack

    See http://swift.openstack.org

    Swift requires environment variables (OS_*) for the authentication and configuration"""

    def __init__(self, container, prefix=''):
        self.container = container
        self.prefix = prefix
        self.client = SwiftService()

    def download(self, source, target):
        objects = [self.prefix + '/' + source]
        options = {'out_file': target}
        return list(self.client.download(self.container, objects, options))

    def upload(self, source, target):
        object_name = self.prefix + '/' + target
        objects = [SwiftUploadObject(source, object_name=object_name)]
        return list(self.client.upload(self.container, objects))

    def ls(self, path):
        fpath = self.prefix + '/' + path + '/'
        clisting = self.client.list(self.container, {'prefix': fpath})
        listing = list(clisting)[0]['listing']
        result = [d['name'].replace(fpath, '') for d in listing]
        return result

    def url(self, path=''):
        return self.container + '/' + self.prefix + '/' + path

    def test_upload_object_job_file(self):
        # Uploading a file results in the file object being wrapped in a
        # LengthWrapper. This test sets the options in such a way that much
        # of _upload_object_job is skipped bringing the critical path down
        # to around 60 lines to ease testing.
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            expected_r = {
                'action': 'upload_object',
                'attempts': 2,
                'container': 'test_c',
                'headers': {},
                'large_object': False,
                'object': 'test_o',
                'response_dict': {},
                'status': 'uploaded',
                'success': True,
            }
            expected_mtime = float(os.path.getmtime(f.name))

            mock_conn = mock.Mock()
            mock_conn.put_object.return_value = ''
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            mtime = float(r['headers']['x-object-meta-mtime'])
            self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
            del r['headers']['x-object-meta-mtime']

            self.assertEqual(r['path'], f.name)
            del r['path']

            self._assertDictEqual(r, expected_r)
            self.assertEqual(mock_conn.put_object.call_count, 1)
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=30,
                                                    headers={},
                                                    response_dict={})
            contents = mock_conn.put_object.call_args[0][2]
            self.assertIsInstance(contents, utils.LengthWrapper)
            self.assertEqual(len(contents), 30)
            # This read forces the LengthWrapper to calculate the md5
            # for the read content. This also checks that LengthWrapper was
            # initialized with md5=True
            self.assertEqual(contents.read(), b'a' * 30)
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())

def upload(source, dest):
    """
    Upload file to a remote SWIFT store
    @param source: List of local source path to upload from.
    @type source : Dicrionary
    @param dest: The destination information such as destination url to upload the file.
    @type dest: Dictionary
    @return: True if upload is successful. Otherwise False.
    """

    url = urlsplit(dest['url'])

    _, ver, account, container, object_name = url.path.split('/', 4)

    swift_opts = {
        'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
            scheme=re.sub(r'^swift\+', '', url.scheme),
            netloc=url.netloc,
            ver=ver,
            account=account)
    }
    # SwiftService knows about environment variables
    for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
        if opt in dest:
            swift_opts[opt] = dest[opt]
    try:
        swift = SwiftService(swift_opts)
        headers = []
        if 'content_type' in source:
            headers.append('Content-Type: {}'.format(source['content_type']))

        retries = 5  # number of retries left
        backoff = 30  # wait time between retries
        backoff_inc = 30  # increase in wait time per retry

        while retries:
            retries -= 1
            try:
                for result in swift.upload(container, [SwiftUploadObject(source['url'], object_name=object_name, options={'header': headers})]):
                    # TODO: we may get  result['action'] = 'create_container'
                    # self.assertNotIn(member, container)d result['action'] = 'upload_object';  result['path'] =
                    # source['url']
                    if not result['success']:
                        raise Exception(
                            'Upload to Swift {container}/{object_name} failed with {error}'.format(object_name=object_name, **result))
                # no exception we can continue
                retries = 0
            except Exception as e:
                if not retries:
                    # reraise if no retries left
                    raise
                LOG.warn('Upload to Swift failed: %s - %d retries left', e, retries)
                time.sleep(backoff)
                backoff += backoff_inc
                backoff_inc += 30
    except Exception as e:
        LOG.error("Upload to swift failed: %s", e, exc_info=True)
        raise

 def test_upload_with_bad_segment_size(self):
     for bad in ('ten', '1234X', '100.3'):
         options = {'segment_size': bad}
         try:
             service = SwiftService(options)
             next(service.upload('c', 'o'))
             self.fail('Expected SwiftError when segment_size=%s' % bad)
         except SwiftError as exc:
             self.assertEqual('Segment size should be an integer value',
                              exc.value)

class DiracStore(object):
    def __init__(self, container, topic, kafka):
        self.container = container
        self.swift = SwiftService()
        self.out = OutputManager()
        self.kafka = kafka

        auth = get_auth()
        self.url = auth[0]
        self.token = auth[1]
        
        self.topic = topic
        self.producer = KafkaProducer(bootstrap_servers=kafka)

    def send(self, resource):
        json = message_to_json(resource)
        print("sending " + json)
        self.producer.send(self.topic, json.encode('utf-8'))

    def make_resource(self, stat):
        resource = schema.Resource()
        resource.type = 'file'
        resource.name = stat['Object']
        resource.location = self.url + '/' + self.container + '/' + stat['Object']
        resource.mimeType = stat['Content Type']
        resource.size = long(stat['Content Length'])
        resource.created = stat['Last Modified']

        return resource

    def stat(self, paths):
        stat = {}
        for response in self.swift.stat(container=self.container, objects=paths):
            if response['success']:
                stat[response['object']] = {item[0]: item[1] for item in response['items']}

        return stat

    def store(self, path, source):
        isdir = os.path.isdir(source)
        base = source if isdir else os.path.dirname(source)
        sources = os.listdir(source) if isdir else [source]
        locations = [os.path.join(path, os.path.basename(file)) for file in sources]
        print(str(len(locations)) + " locations!")
        stats = self.stat(locations)
        objs = [SwiftUploadObject(os.path.join(base, os.path.basename(location)), object_name=location) for location in locations if not location in stats]
        print(str(len(objs)) + " previously unseen!")

        for response in self.swift.upload(self.container, objs):
            if response['success']:
                if 'object' in response:
                    print('uploading ' + response['object'])
                    stat = self.stat([response['object']])
                    resource = self.make_resource(stat.values()[0])
                    self.send(resource)

    def test_upload_object_job_stream(self):
        # Streams are wrapped as ReadableToIterable
        with tempfile.TemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            f.seek(0)
            expected_r = {
                'action': 'upload_object',
                'attempts': 2,
                'container': 'test_c',
                'headers': {},
                'large_object': False,
                'object': 'test_o',
                'response_dict': {},
                'status': 'uploaded',
                'success': True,
                'path': None,
            }
            expected_mtime = float(time.time())

            mock_conn = mock.Mock()
            mock_conn.put_object.return_value = ''
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            mtime = float(r['headers']['x-object-meta-mtime'])
            self.assertAlmostEqual(mtime, expected_mtime, delta=0.5)
            del r['headers']['x-object-meta-mtime']

            self._assertDictEqual(r, expected_r)
            self.assertEqual(mock_conn.put_object.call_count, 1)
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=None,
                                                    headers={},
                                                    response_dict={})
            contents = mock_conn.put_object.call_args[0][2]
            self.assertIsInstance(contents, utils.ReadableToIterable)
            self.assertEqual(contents.chunk_size, 65536)
            # next retrieves the first chunk of the stream or len(chunk_size)
            # or less, it also forces the md5 to be calculated.
            self.assertEqual(next(contents), b'a' * 30)
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())

    def test_upload_segment_job(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 10)
            f.write(b'b' * 10)
            f.write(b'c' * 10)
            f.flush()

            # Mock the connection to return an empty etag. This
            # skips etag validation which would fail as the LengthWrapper
            # isnt read from.
            mock_conn = mock.Mock()
            mock_conn.put_object.return_value = ''
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)
            expected_r = {
                'action': 'upload_segment',
                'for_object': 'test_o',
                'segment_index': 2,
                'segment_size': 10,
                'segment_location': '/test_c_segments/test_s_1',
                'log_line': 'test_o segment 2',
                'success': True,
                'response_dict': {},
                'segment_etag': '',
                'attempts': 2,
            }

            s = SwiftService()
            r = s._upload_segment_job(conn=mock_conn,
                                      path=f.name,
                                      container='test_c',
                                      segment_name='test_s_1',
                                      segment_start=10,
                                      segment_size=10,
                                      segment_index=2,
                                      obj_name='test_o',
                                      options={'segment_container': None,
                                               'checksum': True})

            self._assertDictEqual(r, expected_r)

            self.assertEqual(mock_conn.put_object.call_count, 1)
            mock_conn.put_object.assert_called_with('test_c_segments',
                                                    'test_s_1',
                                                    mock.ANY,
                                                    content_length=10,
                                                    response_dict={})
            contents = mock_conn.put_object.call_args[0][2]
            self.assertIsInstance(contents, utils.LengthWrapper)
            self.assertEqual(len(contents), 10)
            # This read forces the LengthWrapper to calculate the md5
            # for the read content.
            self.assertEqual(contents.read(), b'b' * 10)
            self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest())

    def test_upload_object_job_identical_dlo(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            segment_etag = md5(b'a' * 10).hexdigest()

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'x-object-manifest': 'test_c_segments/test_o/prefix',
                'content-length': 30,
                'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()}
            mock_conn.get_container.side_effect = [
                (None, [{"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/00"},
                        {"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/01"}]),
                (None, [{"bytes": 10, "hash": segment_etag,
                         "name": "test_o/prefix/02"}]),
                (None, {})]
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            with mock.patch('swiftclient.service.get_conn',
                            return_value=mock_conn):
                r = s._upload_object_job(conn=mock_conn,
                                         container='test_c',
                                         source=f.name,
                                         obj='test_o',
                                         options={'changed': False,
                                                  'skip_identical': True,
                                                  'leave_segments': True,
                                                  'header': '',
                                                  'segment_size': 10})

            self.assertIsNone(r.get('error'))
            self.assertTrue(r['success'])
            self.assertEqual('skipped-identical', r.get('status'))
            self.assertEqual(0, mock_conn.put_object.call_count)
            self.assertEqual(1, mock_conn.head_object.call_count)
            self.assertEqual(3, mock_conn.get_container.call_count)
            mock_conn.head_object.assert_called_with('test_c', 'test_o')
            expected = [
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker='', delimiter=None),
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker="test_o/prefix/01", delimiter=None),
                mock.call('test_c_segments', prefix='test_o/prefix',
                          marker="test_o/prefix/02", delimiter=None),
            ]
            mock_conn.get_container.assert_has_calls(expected)

    def test_upload_object_job_identical_slo_with_nesting(self):
        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()
            seg_etag = md5(b'a' * 10).hexdigest()
            submanifest = "[%s]" % ",".join(
                ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2)
            submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest()
            manifest = "[%s]" % ",".join([
                '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",'
                '"bytes":20,"hash":"%s"}' % submanifest_etag,
                '{"bytes":10,"hash":"%s"}' % seg_etag])

            mock_conn = mock.Mock()
            mock_conn.head_object.return_value = {
                'x-static-large-object': True,
                'content-length': 30,
                'etag': md5(submanifest_etag.encode('ascii') +
                            seg_etag.encode('ascii')).hexdigest()}
            mock_conn.get_object.side_effect = [
                ({}, manifest.encode('ascii')),
                ({}, submanifest.encode('ascii'))]
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': True,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 10})

            self.assertIsNone(r.get('error'))
            self.assertTrue(r['success'])
            self.assertEqual('skipped-identical', r.get('status'))
            self.assertEqual(0, mock_conn.put_object.call_count)
            self.assertEqual([mock.call('test_c', 'test_o')],
                             mock_conn.head_object.mock_calls)
            self.assertEqual([
                mock.call('test_c', 'test_o',
                          query_string='multipart-manifest=get'),
                mock.call('test_c_segments', 'test_sub_slo',
                          query_string='multipart-manifest=get'),
            ], mock_conn.get_object.mock_calls)

    def test_delete_empty_container_excpetion(self):
        mock_conn = self._get_mock_connection()
        mock_conn.delete_container = Mock(side_effect=self.exc)
        expected_r = self._get_expected({
            'action': 'delete_container',
            'success': False,
            'object': None,
            'error': self.exc
        })

        s = SwiftService()
        r = s._delete_empty_container(mock_conn, 'test_c')

        mock_conn.delete_container.assert_called_once_with(
            'test_c', response_dict={}
        )
        self._assertDictEqual(expected_r, r)

    def test_delete_object(self):
        mock_q = Queue()
        mock_conn = self._get_mock_connection()
        mock_conn.head_object = Mock(return_value={})
        expected_r = self._get_expected({
            'action': 'delete_object',
            'success': True
        })

        s = SwiftService()
        r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)

        mock_conn.head_object.assert_called_once_with('test_c', 'test_o')
        mock_conn.delete_object.assert_called_once_with(
            'test_c', 'test_o', query_string=None, response_dict={}
        )
        self._assertDictEqual(expected_r, r)

    def test_upload_object_job_etag_mismatch(self):
        # The etag test for both streams and files use the same code
        # so only one test should be needed.
        def _consuming_conn(*a, **kw):
            contents = a[2]
            contents.read()  # Force md5 calculation
            return 'badresponseetag'

        with tempfile.NamedTemporaryFile() as f:
            f.write(b'a' * 30)
            f.flush()

            mock_conn = mock.Mock()
            mock_conn.put_object.side_effect = _consuming_conn
            type(mock_conn).attempts = mock.PropertyMock(return_value=2)

            s = SwiftService()
            r = s._upload_object_job(conn=mock_conn,
                                     container='test_c',
                                     source=f.name,
                                     obj='test_o',
                                     options={'changed': False,
                                              'skip_identical': False,
                                              'leave_segments': True,
                                              'header': '',
                                              'segment_size': 0,
                                              'checksum': True})

            self.assertEqual(r['success'], False)
            self.assertIn('error', r)
            self.assertIn('md5 mismatch', str(r['error']))

            self.assertEqual(mock_conn.put_object.call_count, 1)
            expected_headers = {'x-object-meta-mtime': mock.ANY}
            mock_conn.put_object.assert_called_with('test_c', 'test_o',
                                                    mock.ANY,
                                                    content_length=30,
                                                    headers=expected_headers,
                                                    response_dict={})

            contents = mock_conn.put_object.call_args[0][2]
            self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())