Python ContainerBroker.initialize方法代码示例

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def get_reconciler_broker(self, timestamp):
        """
        Get a local instance of the reconciler container broker that is
        appropriate to enqueue the given timestamp.

        :param timestamp: the timestamp of the row to be enqueued

        :returns: a local reconciler broker
        """
        container = get_reconciler_container_name(timestamp)
        if self.reconciler_containers and \
                container in self.reconciler_containers:
            return self.reconciler_containers[container][1]
        account = MISPLACED_OBJECTS_ACCOUNT
        part = self.ring.get_part(account, container)
        node = self.find_local_handoff_for_part(part)
        if not node:
            raise DeviceUnavailable(
                'No mounted devices found suitable to Handoff reconciler '
                'container %s in partition %s' % (container, part))
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, node['device'], db_dir, hsh + '.db')
        broker = ContainerBroker(db_path, account=account, container=container)
        if not os.path.exists(broker.db_file):
            try:
                broker.initialize(timestamp, 0)
            except DatabaseAlreadyExists:
                pass
        if self.reconciler_containers is not None:
            self.reconciler_containers[container] = part, broker, node['id']
        return broker

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
 def test_find_replace_enable(self):
     db_file = os.path.join(self.testdir, 'hash.db')
     broker = ContainerBroker(db_file)
     broker.account = 'a'
     broker.container = 'c'
     broker.initialize()
     ts = utils.Timestamp.now()
     broker.merge_items([
         {'name': 'obj%02d' % i, 'created_at': ts.internal, 'size': 0,
          'content_type': 'application/octet-stream', 'etag': 'not-really',
          'deleted': 0, 'storage_policy_index': 0,
          'ctype_timestamp': ts.internal, 'meta_timestamp': ts.internal}
         for i in range(100)])
     out = StringIO()
     err = StringIO()
     with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
         with mock_timestamp_now() as now:
             main([broker.db_file, 'find_and_replace', '10', '--enable'])
     expected = [
         'No shard ranges found to delete.',
         'Injected 10 shard ranges.',
         'Run container-replicator to replicate them to other nodes.',
         "Container moved to state 'sharding' with epoch %s." %
         now.internal,
         'Run container-sharder on all nodes to shard the container.']
     self.assertEqual(expected, out.getvalue().splitlines())
     self.assertEqual(['Loaded db broker for a/c.'],
                      err.getvalue().splitlines())
     self._assert_enabled(broker, now)
     self.assertEqual(
         [(data['lower'], data['upper']) for data in self.shard_data],
         [(sr.lower_str, sr.upper_str) for sr in broker.get_shard_ranges()])

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def _gen_container_stat(self, set_metadata=False, drop_metadata=False):
        if set_metadata:
            self.conf['metadata_keys'] = 'test1,test2'
            # webob runs title on all headers
        stat = db_stats_collector.ContainerStatsCollector(self.conf)
        output_data = set()
        for i in range(10):
            cont_db = ContainerBroker(
                "%s/container-stats-201001010%s-%s.db" % (self.containers, i,
                                                          uuid.uuid4().hex),
                 account='test_acc_%s' % i, container='test_con')
            cont_db.initialize()
            cont_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
            metadata_output = ''
            if set_metadata:
                if i % 2:
                    cont_db.update_metadata({'X-Container-Meta-Test1': (5, 1)})
                    metadata_output = ',1,'
                else:
                    cont_db.update_metadata({'X-Container-Meta-Test2': (7, 2)})
                    metadata_output = ',,1'
            # this will "commit" the data
            cont_db.get_info()
            if drop_metadata:
                output_data.add('''"test_acc_%s","test_con",1,10,,''' % i)
            else:
                output_data.add('''"test_acc_%s","test_con",1,10%s''' %
                                (i, metadata_output))
            if drop_metadata:
                self._drop_metadata_col(cont_db, 'test_acc_%s' % i)

        self.assertEqual(len(output_data), 10)
        return stat, output_data

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
 def test_container_stat_get_data(self):
     stat = db_stats_collector.ContainerStatsCollector(self.conf)
     container_db = ContainerBroker("%s/con.db" % self.containers,
                                  account='test_acc', container='test_con')
     container_db.initialize()
     container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
     info = stat.get_data("%s/con.db" % self.containers)
     self.assertEquals('''"test_acc","test_con",1,10\n''', info)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_unicode(self):
        cu = container_updater.ContainerUpdater(
            {
                "devices": self.devices_dir,
                "mount_check": "false",
                "swift_dir": self.testdir,
                "interval": "1",
                "concurrency": "1",
                "node_timeout": "15",
            }
        )
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        subdir = os.path.join(containers_dir, "subdir")
        os.mkdir(subdir)
        cb = ContainerBroker(os.path.join(subdir, "hash.db"), account="a", container="\xce\xa9")
        cb.initialize(normalize_timestamp(1), 0)
        cb.put_object("\xce\xa9", normalize_timestamp(2), 3, "text/plain", "68b329da9893e34099c7d8ad5cb9c940")

        def accept(sock, addr):
            try:
                with Timeout(3):
                    inc = sock.makefile("rb")
                    out = sock.makefile("wb")
                    out.write("HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n")
                    out.flush()
                    inc.read()
            except BaseException as err:
                import traceback

                traceback.print_exc()
                return err
            return None

        bindsock = listen(("127.0.0.1", 0))

        def spawn_accepts():
            events = []
            for _junk in range(2):
                with Timeout(3):
                    sock, addr = bindsock.accept()
                    events.append(spawn(accept, sock, addr))
            return events

        spawned = spawn(spawn_accepts)
        for dev in cu.get_account_ring().devs:
            if dev is not None:
                dev["port"] = bindsock.getsockname()[1]
        cu.run_once()
        for event in spawned.wait():
            err = event.wait()
            if err:
                raise err
        info = cb.get_info()
        self.assertEqual(info["object_count"], 1)
        self.assertEqual(info["bytes_used"], 3)
        self.assertEqual(info["reported_object_count"], 1)
        self.assertEqual(info["reported_bytes_used"], 3)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_unicode(self):
        cu = container_updater.ContainerUpdater({
            'devices': self.devices_dir,
            'mount_check': 'false',
            'swift_dir': self.testdir,
            'interval': '1',
            'concurrency': '1',
            'node_timeout': '15',
        })
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(subdir)
        cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
                             container='\xce\xa9')
        cb.initialize(normalize_timestamp(1), 0)
        cb.put_object('\xce\xa9', normalize_timestamp(2), 3, 'text/plain',
                      '68b329da9893e34099c7d8ad5cb9c940')

        def accept(sock, addr):
            try:
                with Timeout(3):
                    inc = sock.makefile('rb')
                    out = sock.makefile('wb')
                    out.write('HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n')
                    out.flush()
                    inc.read()
            except BaseException as err:
                import traceback
                traceback.print_exc()
                return err
            return None

        bindsock = listen(('127.0.0.1', 0))

        def spawn_accepts():
            events = []
            for _junk in range(2):
                with Timeout(3):
                    sock, addr = bindsock.accept()
                    events.append(spawn(accept, sock, addr))
            return events

        spawned = spawn(spawn_accepts)
        for dev in cu.get_account_ring().devs:
            if dev is not None:
                dev['port'] = bindsock.getsockname()[1]
        cu.run_once()
        for event in spawned.wait():
            err = event.wait()
            if err:
                raise err
        info = cb.get_info()
        self.assertEquals(info['object_count'], 1)
        self.assertEquals(info['bytes_used'], 3)
        self.assertEquals(info['reported_object_count'], 1)
        self.assertEquals(info['reported_bytes_used'], 3)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
 def _make_broker(self, account='a', container='c',
                  device='sda', part=0):
     datadir = os.path.join(
         self.testdir, device, 'containers', str(part), 'ash', 'hash')
     db_file = os.path.join(datadir, 'hash.db')
     broker = ContainerBroker(
         db_file, account=account, container=container)
     broker.initialize()
     return broker

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
 def test_container_stat_get_metadata(self):
     container_db = ContainerBroker("%s/con.db" % self.containers,
                                  account='test_acc', container='test_con')
     container_db.initialize(storage_policy_index=0)
     container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag')
     container_db.update_metadata({'X-Container-Meta-Test1': ('val', 1000)})
     self.conf['metadata_keys'] = 'test1,test2'
     stat = db_stats_collector.ContainerStatsCollector(self.conf)
     info = stat.get_data("%s/con.db" % self.containers)
     self.assertEquals('''"test_acc","test_con",1,10,1,\n''', info)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_unicode(self):
        cu = self._get_container_updater()
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(subdir)
        cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
                             container='\xce\xa9')
        cb.initialize(normalize_timestamp(1), 0)
        obj_name = u'\N{GREEK CAPITAL LETTER OMEGA}'
        if six.PY2:
            obj_name = obj_name.encode('utf-8')
        cb.put_object(obj_name, normalize_timestamp(2), 3, 'text/plain',
                      '68b329da9893e34099c7d8ad5cb9c940')

        def accept(sock, addr):
            try:
                with Timeout(3):
                    inc = sock.makefile('rb')
                    out = sock.makefile('wb')
                    out.write(b'HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n')
                    out.flush()
                    inc.read()
            except BaseException as err:
                import traceback
                traceback.print_exc()
                return err
            return None

        bindsock = listen_zero()

        def spawn_accepts():
            events = []
            for _junk in range(2):
                with Timeout(3):
                    sock, addr = bindsock.accept()
                    events.append(spawn(accept, sock, addr))
            return events

        spawned = spawn(spawn_accepts)
        for dev in cu.get_account_ring().devs:
            if dev is not None:
                dev['port'] = bindsock.getsockname()[1]
        cu.run_once()
        for event in spawned.wait():
            err = event.wait()
            if err:
                raise err
        info = cb.get_info()
        self.assertEqual(info['object_count'], 1)
        self.assertEqual(info['bytes_used'], 3)
        self.assertEqual(info['reported_object_count'], 1)
        self.assertEqual(info['reported_bytes_used'], 3)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_run_once_with_get_info_timeout(self, mock_dump_recon):
        cu = self._get_container_updater()
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(subdir)
        db_file = os.path.join(subdir, 'hash.db')
        cb = ContainerBroker(db_file, account='a', container='c')
        cb.initialize(normalize_timestamp(1), 0)

        timeout = exceptions.LockTimeout(10, db_file)
        timeout.cancel()
        with mock.patch('swift.container.updater.ContainerBroker.get_info',
                        side_effect=timeout):
            cu.run_once()
        log_lines = self.logger.get_lines_for_level('info')
        self.assertIn('Failed to get container info (Lock timeout: '
                      '10 seconds: %s); skipping.' % db_file, log_lines)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_error_in_process(self, mock_process, mock_dump_recon):
        cu = self._get_container_updater()
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(subdir)
        cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
                             container='c', pending_timeout=1)
        cb.initialize(normalize_timestamp(1), 0)

        cu.run_once()

        log_lines = self.logger.get_lines_for_level('error')
        self.assertTrue(log_lines)
        self.assertIn('Error processing container ', log_lines[0])
        self.assertIn('devices/sda1/containers/subdir/hash.db', log_lines[0])
        self.assertIn('Boom!', log_lines[0])
        self.assertFalse(log_lines[1:])
        self.assertEqual(1, len(mock_dump_recon.mock_calls))

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def setUp(self):
        self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater')
        rmtree(self.testdir, ignore_errors=1)
        os.mkdir(self.testdir)
        self.devices_dir = os.path.join(self.testdir, 'devices')
        os.mkdir(self.devices_dir)
        self.sda1 = os.path.join(self.devices_dir, 'sda1')
        os.mkdir(self.sda1)

        containers_dir = os.path.join(self.sda1, container_server.DATADIR)
        os.mkdir(containers_dir)

        self.assert_(os.path.exists(containers_dir))
        self.subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(self.subdir)
        cb = ContainerBroker(os.path.join(self.subdir, 'hash.db'), account='a',
                             container='c')
        cb.initialize(normalize_timestamp(1))
        cb.put_object('o', normalize_timestamp(2), 3, 'text/plain',
                '68b329da9893e34099c7d8ad5cb9c940')

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_find_shard_ranges(self):
        db_file = os.path.join(self.testdir, 'hash.db')
        broker = ContainerBroker(db_file)
        broker.account = 'a'
        broker.container = 'c'
        broker.initialize()
        ts = utils.Timestamp.now()
        broker.merge_items([
            {'name': 'obj%02d' % i, 'created_at': ts.internal, 'size': 0,
             'content_type': 'application/octet-stream', 'etag': 'not-really',
             'deleted': 0, 'storage_policy_index': 0,
             'ctype_timestamp': ts.internal, 'meta_timestamp': ts.internal}
            for i in range(100)])

        # Default uses a large enough value that sharding isn't required
        out = StringIO()
        err = StringIO()
        with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
            main([db_file, 'find'])
        self.assert_formatted_json(out.getvalue(), [])
        err_lines = err.getvalue().split('\n')
        self.assert_starts_with(err_lines[0], 'Loaded db broker for ')
        self.assert_starts_with(err_lines[1], 'Found 0 ranges in ')

        out = StringIO()
        err = StringIO()
        with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
            main([db_file, 'find', '100'])
        self.assert_formatted_json(out.getvalue(), [])
        err_lines = err.getvalue().split('\n')
        self.assert_starts_with(err_lines[0], 'Loaded db broker for ')
        self.assert_starts_with(err_lines[1], 'Found 0 ranges in ')

        out = StringIO()
        err = StringIO()
        with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
            main([db_file, 'find', '99'])
        self.assert_formatted_json(out.getvalue(), [
            {'index': 0, 'lower': '', 'upper': 'obj98', 'object_count': 99},
            {'index': 1, 'lower': 'obj98', 'upper': '', 'object_count': 1},
        ])
        err_lines = err.getvalue().split('\n')
        self.assert_starts_with(err_lines[0], 'Loaded db broker for ')
        self.assert_starts_with(err_lines[1], 'Found 2 ranges in ')

        out = StringIO()
        err = StringIO()
        with mock.patch('sys.stdout', out), mock.patch('sys.stderr', err):
            main([db_file, 'find', '10'])
        self.assert_formatted_json(out.getvalue(), [
            {'index': 0, 'lower': '', 'upper': 'obj09', 'object_count': 10},
            {'index': 1, 'lower': 'obj09', 'upper': 'obj19',
             'object_count': 10},
            {'index': 2, 'lower': 'obj19', 'upper': 'obj29',
             'object_count': 10},
            {'index': 3, 'lower': 'obj29', 'upper': 'obj39',
             'object_count': 10},
            {'index': 4, 'lower': 'obj39', 'upper': 'obj49',
             'object_count': 10},
            {'index': 5, 'lower': 'obj49', 'upper': 'obj59',
             'object_count': 10},
            {'index': 6, 'lower': 'obj59', 'upper': 'obj69',
             'object_count': 10},
            {'index': 7, 'lower': 'obj69', 'upper': 'obj79',
             'object_count': 10},
            {'index': 8, 'lower': 'obj79', 'upper': 'obj89',
             'object_count': 10},
            {'index': 9, 'lower': 'obj89', 'upper': '', 'object_count': 10},
        ])
        err_lines = err.getvalue().split('\n')
        self.assert_starts_with(err_lines[0], 'Loaded db broker for ')
        self.assert_starts_with(err_lines[1], 'Found 10 ranges in ')

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
    def test_run_once(self):
        cu = self._get_container_updater()
        cu.run_once()
        containers_dir = os.path.join(self.sda1, DATADIR)
        os.mkdir(containers_dir)
        cu.run_once()
        self.assertTrue(os.path.exists(containers_dir))
        subdir = os.path.join(containers_dir, 'subdir')
        os.mkdir(subdir)
        cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a',
                             container='c')
        cb.initialize(normalize_timestamp(1), 0)
        cu.run_once()
        info = cb.get_info()
        self.assertEqual(info['object_count'], 0)
        self.assertEqual(info['bytes_used'], 0)
        self.assertEqual(info['reported_object_count'], 0)
        self.assertEqual(info['reported_bytes_used'], 0)

        cb.put_object('o', normalize_timestamp(2), 3, 'text/plain',
                      '68b329da9893e34099c7d8ad5cb9c940')
        cu.run_once()
        info = cb.get_info()
        self.assertEqual(info['object_count'], 1)
        self.assertEqual(info['bytes_used'], 3)
        self.assertEqual(info['reported_object_count'], 0)
        self.assertEqual(info['reported_bytes_used'], 0)

        def accept(sock, addr, return_code):
            try:
                with Timeout(3):
                    inc = sock.makefile('rb')
                    out = sock.makefile('wb')
                    out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' %
                              return_code)
                    out.flush()
                    self.assertEqual(inc.readline(),
                                     'PUT /sda1/0/a/c HTTP/1.1\r\n')
                    headers = {}
                    line = inc.readline()
                    while line and line != '\r\n':
                        headers[line.split(':')[0].lower()] = \
                            line.split(':')[1].strip()
                        line = inc.readline()
                    self.assertTrue('x-put-timestamp' in headers)
                    self.assertTrue('x-delete-timestamp' in headers)
                    self.assertTrue('x-object-count' in headers)
                    self.assertTrue('x-bytes-used' in headers)
            except BaseException as err:
                import traceback
                traceback.print_exc()
                return err
            return None
        bindsock = listen_zero()

        def spawn_accepts():
            events = []
            for _junk in range(2):
                sock, addr = bindsock.accept()
                events.append(spawn(accept, sock, addr, 201))
            return events

        spawned = spawn(spawn_accepts)
        for dev in cu.get_account_ring().devs:
            if dev is not None:
                dev['port'] = bindsock.getsockname()[1]
        cu.run_once()
        for event in spawned.wait():
            err = event.wait()
            if err:
                raise err
        info = cb.get_info()
        self.assertEqual(info['object_count'], 1)
        self.assertEqual(info['bytes_used'], 3)
        self.assertEqual(info['reported_object_count'], 1)
        self.assertEqual(info['reported_bytes_used'], 3)

# 需要导入模块: from swift.container.backend import ContainerBroker [as 别名]
# 或者: from swift.container.backend.ContainerBroker import initialize [as 别名]
from swift.container.backend import ContainerBroker
from swift.common.utils import Timestamp
from time import time
import os


def show_result(is_deleted, is_empty):
    print 'deleted? %s\nempty? %s\n' % (is_deleted, is_empty)


if __name__ == '__main__':
    db_file = ':memory:'
    broker = ContainerBroker(db_file=db_file,
                             account='account',
                             container='container')
    broker.initialize(Timestamp(time()).internal, 0)

    obj = 'object'

    print 'put object:'
    broker.put_object(obj, Timestamp(time()).internal, 10, '', '')

    info, is_deleted = broker.get_info_is_deleted()
    show_result(is_deleted, broker.empty())

    print 'delete object:'
    broker.delete_object(obj, Timestamp(time()).internal)
    info, is_deleted = broker.get_info_is_deleted()
    show_result(is_deleted, broker.empty())

    print 'delete db:'