Python ring.RingBuilder类代码示例

    def create():
        """
swift-ring-builder <builder_file> create <part_power> <replicas>
                                         <min_part_hours>
    Creates <builder_file> with 2^<part_power> partitions and <replicas>.
    <min_part_hours> is number of hours to restrict moving a partition more
    than once.
        """
        if len(argv) < 6:
            print(Commands.create.__doc__.strip())
            exit(EXIT_ERROR)
        # 1、生成RingBuilder对象实例
        builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))

        # 2、创建备份目录
        backup_dir = pathjoin(dirname(builder_file), 'backups')
        try:
            mkdir(backup_dir)
        except OSError as err:
            if err.errno != EEXIST:
                raise

        # 3、保存原始数据到备份目录，以及/etc/swift目录中
        builder.save(pathjoin(backup_dir,
                              '%d.' % time() + basename(builder_file)))
        builder.save(builder_file)
        exit(EXIT_SUCCESS)

    def test_load(self):
        rb = ring.RingBuilder(8, 3, 1)
        devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
                 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1',
                 'meta': 'meta0'},
                {'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
                 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1',
                 'meta': 'meta1'},
                {'id': 2, 'region': 0, 'zone': 2, 'weight': 2,
                 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1',
                 'meta': 'meta2'},
                {'id': 3, 'region': 0, 'zone': 3, 'weight': 2,
                 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}]
        for d in devs:
            rb.add_dev(d)
        rb.rebalance()

        real_pickle = pickle.load
        try:
            #test a legit builder
            fake_pickle = Mock(return_value=rb)
            fake_open = Mock(return_value=None)
            pickle.load = fake_pickle
            builder = RingBuilder.load('fake.builder', open=fake_open)
            self.assertEquals(fake_pickle.call_count, 1)
            fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
            self.assertEquals(builder, rb)
            fake_pickle.reset_mock()
            fake_open.reset_mock()

            #test old style builder
            fake_pickle.return_value = rb.to_dict()
            pickle.load = fake_pickle
            builder = RingBuilder.load('fake.builder', open=fake_open)
            fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
            self.assertEquals(builder.devs, rb.devs)
            fake_pickle.reset_mock()
            fake_open.reset_mock()

            #test old devs but no meta
            no_meta_builder = rb
            for dev in no_meta_builder.devs:
                del(dev['meta'])
            fake_pickle.return_value = no_meta_builder
            pickle.load = fake_pickle
            builder = RingBuilder.load('fake.builder', open=fake_open)
            fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
            self.assertEquals(builder.devs, rb.devs)
            fake_pickle.reset_mock()
        finally:
            pickle.load = real_pickle

    def test_set_info(self):
        for search_value in self.search_values:

            self.create_sample_ring()
            argv = ["", self.tmpfile, "set_info", search_value,
                    "127.0.1.1:8000/sda1_other meta data"]
            self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)

            # Check that device was created with given data
            ring = RingBuilder.load(self.tmpfile)
            dev = [d for d in ring.devs if d['id'] == 0][0]
            self.assertEqual(dev['ip'], '127.0.1.1')
            self.assertEqual(dev['port'], 8000)
            self.assertEqual(dev['device'], 'sda1')
            self.assertEqual(dev['meta'], 'other meta data')

            # Check that second device in ring is not affected
            dev = [d for d in ring.devs if d['id'] == 1][0]
            self.assertEqual(dev['ip'], '127.0.0.2')
            self.assertEqual(dev['port'], 6001)
            self.assertEqual(dev['device'], 'sda2')
            self.assertEqual(dev['meta'], '')

            # Final check, rebalance and check ring is ok
            ring.rebalance()
            self.assertTrue(ring.validate())

 def __init__(self, builder_file, swiftdir='/etc/swift',
              backup_dir_name='backups', verbose=False):
     self.verbose = verbose
     self.builder_file = builder_file
     self.swiftdir = swiftdir
     self.backup_dir = pathjoin(self.swiftdir, backup_dir_name)
     self.builder = RingBuilder.load(builder_file)

 def add_to_ring(self, builder_type, body, lasthash, start_response, env):
     """ Handle a add device post """
     with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
         self.verify_current_hash(self.bf_path[builder_type], lasthash)
         builder = RingBuilder.load(self.bf_path[builder_type])
         ring_modified = False
         try:
             for device in body['devices']:
                 sleep()  # so we don't starve/block
                 if not self._is_existing_dev(builder, device['ip'],
                                              int(device['port']),
                                              device['device']):
                     self._add_device(builder, int(device['zone']),
                                      device['ip'], int(device['port']),
                                      device['device'],
                                      float(device['weight']),
                                      device['meta'])
                     ring_modified = True
         except (AttributeError, KeyError, ValueError, TypeError) as err:
             return self.return_response(False, lasthash,
                                         "Malformed request.",
                                         start_response, env)
         if ring_modified:
             newmd5 = self.write_builder(builder,
                                         self.bf_path[builder_type])
             return self.return_response(True, newmd5, None,
                                         start_response, env)
         else:
             return self.return_response(False, lasthash,
                                         'Ring remains unchanged.',
                                         start_response, env)

    def change_meta(self, builder_type, dev_meta, lasthash, start_response,
                    env):
        """ Change meta info for devices

        :param builder_type: the builder_type to use when loading the builder
        :param dev_meta: a dict of device id and meta info
        :param lasthash: the hash to use when verifying state
        """
        with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
            self.verify_current_hash(self.bf_path[builder_type], lasthash)
            builder = RingBuilder.load(self.bf_path[builder_type])
            try:
                modified = False
                for dev_id in dev_meta:
                    sleep()  # so we don't starve/block
                    for device in builder.devs:
                        if not device:
                            continue
                        if device['id'] == int(dev_id):
                            modified = True
                            device['meta'] = '%s' % dev_meta[dev_id]
                if modified:
                    newmd5 = self.write_builder(builder,
                                                self.bf_path[builder_type])
                    return self.return_response(True, newmd5, None,
                                                start_response, env)
                else:
                    return self.return_response(False, lasthash,
                                                'Invalid dev id %s.' % dev_id,
                                                start_response, env)
            except ValueError as err:
                return self.return_response(False, lasthash, str(err),
                                            start_response, env)

    def change_weight(self, builder_type, dev_weights, lasthash,
                      start_response, env):
        """ Change weight of devices

        :param builder_type: the builder_type to use when loading the builder
        :param dev_weights: a dict of device id and weight
        :param lasthash: the hash to use when verifying state
        """
        with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
            self.verify_current_hash(self.bf_path[builder_type], lasthash)
            builder = RingBuilder.load(self.bf_path[builder_type])
            for dev_id in dev_weights:
                sleep()  # so we don't starve/block
                try:
                    builder.set_dev_weight(int(dev_id),
                                           float(dev_weights[dev_id]))
                except (IndexError, TypeError):
                    return self.return_response(False, lasthash,
                                                'Invalid dev id %s.' % dev_id,
                                                start_response, env)
                except ValueError as err:
                    return self.return_response(False, lasthash, str(err),
                                                start_response, env)
            newmd5 = self.write_builder(builder, self.bf_path[builder_type])
            return self.return_response(True, newmd5, None, start_response,
                                        env)

    def remove_devs(self, builder_type, devices, lasthash, start_response,
                    env):
        """ remove devices from the builder

        :params builder_type: the builder_type to use when loading the builder
        :params devices: list of device ids to be removed.
        :params lasthash: the hash to use when verifying state
        """
        with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
            self.verify_current_hash(self.bf_path[builder_type], lasthash)
            builder = RingBuilder.load(self.bf_path[builder_type])
            if not isinstance(devices, list):
                return self.return_response(False, lasthash,
                                            'Malformed request.',
                                            start_response, env)
            for dev_id in devices:
                sleep()  # so we don't starve/block
                try:
                    builder.remove_dev(int(dev_id))
                except (IndexError, TypeError):
                    return self.return_response(False, lasthash,
                                                'Invalid dev id %s.' % dev_id,
                                                start_response, env)
                except RingBuilderError as err:
                    return self.return_response(False, lasthash,
                                                'Error removing %s - %s.' %
                                                (dev_id, err),
                                                start_response, env)
                except ValueError as err:
                    return self.return_response(False, lasthash, str(err),
                                                start_response, env)
            newmd5 = self.write_builder(builder, self.bf_path[builder_type])
            return self.return_response(True, newmd5, None, start_response,
                                        env)

    def test_remove_device(self):
        for search_value in self.search_values:
            self.create_sample_ring()
            argv = ["", self.tmpfile, "remove", search_value]
            self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
            ring = RingBuilder.load(self.tmpfile)

            # Check that weight was set to 0
            dev = [d for d in ring.devs if d['id'] == 0][0]
            self.assertEqual(dev['weight'], 0)

            # Check that device is in list of devices to be removed
            dev = [d for d in ring._remove_devs if d['id'] == 0][0]
            self.assertEqual(dev['region'], 0)
            self.assertEqual(dev['zone'], 0)
            self.assertEqual(dev['ip'], '127.0.0.1')
            self.assertEqual(dev['port'], 6000)
            self.assertEqual(dev['device'], 'sda1')
            self.assertEqual(dev['weight'], 0)
            self.assertEqual(dev['replication_ip'], '127.0.0.1')
            self.assertEqual(dev['replication_port'], 6000)
            self.assertEqual(dev['meta'], 'some meta data')

            # Check that second device in ring is not affected
            dev = [d for d in ring.devs if d['id'] == 1][0]
            self.assertEqual(dev['weight'], 100)
            self.assertFalse([d for d in ring._remove_devs if d['id'] == 1])

            # Final check, rebalance and check ring is ok
            ring.rebalance()
            self.assertTrue(ring.validate())

    def rebalance(self, builder_type, lasthash, start_response, env):
        """ rebalance a ring

            note: rebalance doesn't yield.
        """
        with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
            self.verify_current_hash(self.bf_path[builder_type], lasthash)
            builder = RingBuilder.load(self.bf_path[builder_type])
            devs_changed = builder.devs_changed
            try:
                last_balance = builder.get_balance()
                parts, balance = builder.rebalance()
            except RingBuilderError, err:
                self.logger.exception(_("Error during ring validation."))
                return self.return_response(False, None, err.message,
                                            start_response, env)
            if not parts:
                msg = 'Either none need to be assigned or none can be due ' \
                      'to min_part_hours [%s].' % builder.min_part_hours
                self.logger.error(_(msg))
                return self.return_response(False, None, msg, start_response,
                                            env)
            if not devs_changed and abs(last_balance - balance) < 1:
                msg = 'Refusing to save rebalance. Did not change at least 1%.'
                self.logger.error(_(msg))
                return self.return_response(False, None, msg, start_response,
                                            env)
            try:
                builder.validate()
            except RingValidationError, err:
                self.logger.exception(_("Error during ring validation."))
                return self.return_response(False, None, err.message,
                                            start_response, env)

 def test_validate(self):
     self.create_sample_ring()
     ring = RingBuilder.load(self.tmpfile)
     ring.rebalance()
     ring.save(self.tmpfile)
     argv = ["", self.tmpfile, "validate"]
     self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)

 def test_create_ring(self):
     argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"]
     self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
     ring = RingBuilder.load(self.tmpfile)
     self.assertEqual(ring.part_power, 6)
     self.assertEqual(ring.replicas, 3.14159265359)
     self.assertEqual(ring.min_part_hours, 1)

def write_ring(args, devices, builderfile):
    # Make an educated guess about the used port. These are the defaults for
    # TripleO-based deployments in Mitaka
    builder_fname = os.path.basename(builderfile)
    if 'account' in builder_fname:
        port = 6002
    elif 'container' in builder_fname:
        port = 6001
    elif 'object' in builder_fname:
        port = 6000
    else:
        port = 6000

    logging.debug('Set port for new devices to %d' % port)

    if not os.path.isfile(builderfile):
        logging.info(
            '%s not found, creating new builder file', builderfile)
        rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours)
    else:
        logging.info('Using existing builder file %s', builderfile)
        rb = RingBuilder.load(builderfile)

    # Add all missing devices
    for dev in devices:
        _dev = rb.search_devs({'meta': dev['meta']})
        if not _dev:
            dev['weight'] = float(dev.get('size')) / 10**9
            dev['region'] = 1
            dev['zone'] = 1
            dev['port'] = port
            dev['replication_ip'] = dev['ip']
            dev['replication_port'] = dev['port']
            rb.add_dev(dev)
            logging.info('Added device %s / %s', dev['ip'], dev['device'])
        else:
            logging.info(
                'Ignoring existing device %s / %s', dev['ip'], dev['device'])
    try:
        rb.rebalance()
    except RingValidationError as exc:
        logging.error(exc)
    rb.save(builderfile)
    ring_file = os.path.splitext(builderfile)[0] + '.ring.gz'
    ring_data = rb.get_ring()
    ring_data.save(ring_file)
    return [builderfile, ring_file]

def storage_policies(request):
    """
    Creates a storage policy to swift with an specific ring.
    Allows create replication storage policies and erasure code storage policies
    """

    if request.method == "GET":
        try:
            r = get_redis_connection()
        except RedisError:
            return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)

        keys = r.keys("storage-policy:*")
        storage_policy_list = []
        for key in keys:
            storage_policy = r.hgetall(key)
            to_json_bools(storage_policy, 'deprecated', 'default', 'deployed')
            storage_policy['id'] = str(key).split(':')[-1]
            storage_policy['devices'] = json.loads(storage_policy['devices'])
            storage_policy_list.append(storage_policy)
        return JSONResponse(storage_policy_list, status=status.HTTP_200_OK)

    if request.method == "POST":
        try:
            r = get_redis_connection()
        except RedisError:
            return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)

        data = JSONParser().parse(request)

        if data['policy_type'] == 'EC':
            data['replicas'] = int(data['ec_num_data_fragments']) + int(data['ec_num_parity_fragments'])

        try:
            sp_id = str(r.incr('storage-policies:id'))
            key = 'storage-policy:' + sp_id

            ring = RingBuilder(int(data['partition_power']), int(data['replicas']), int(data['time']))
            ring.save(get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, sp_id))

            r.hmset(key, data)
        except:
            return JSONResponse('Error creating the Storage Policy', status=status.HTTP_500_INTERNAL_SERVER_ERROR)

        return JSONResponse('Account created successfully', status=status.HTTP_201_CREATED)

    return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)

    def test_load(self):
        rb = ring.RingBuilder(8, 3, 1)
        devs = [
            {"id": 0, "zone": 0, "weight": 1, "ip": "127.0.0.0", "port": 10000, "device": "sda1", "meta": "meta0"},
            {"id": 1, "zone": 1, "weight": 1, "ip": "127.0.0.1", "port": 10001, "device": "sdb1", "meta": "meta1"},
            {"id": 2, "zone": 2, "weight": 2, "ip": "127.0.0.2", "port": 10002, "device": "sdc1", "meta": "meta2"},
            {"id": 3, "zone": 3, "weight": 2, "ip": "127.0.0.3", "port": 10003, "device": "sdd1"},
        ]
        for d in devs:
            rb.add_dev(d)
        rb.rebalance()

        real_pickle = pickle.load
        try:
            # test a legit builder
            fake_pickle = Mock(return_value=rb)
            fake_open = Mock(return_value=None)
            pickle.load = fake_pickle
            builder = RingBuilder.load("fake.builder", open=fake_open)
            self.assertEquals(fake_pickle.call_count, 1)
            fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
            self.assertEquals(builder, rb)
            fake_pickle.reset_mock()
            fake_open.reset_mock()

            # test old style builder
            fake_pickle.return_value = rb.to_dict()
            pickle.load = fake_pickle
            builder = RingBuilder.load("fake.builder", open=fake_open)
            fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
            self.assertEquals(builder.devs, rb.devs)
            fake_pickle.reset_mock()
            fake_open.reset_mock()

            # test old devs but no meta
            no_meta_builder = rb
            for dev in no_meta_builder.devs:
                del (dev["meta"])
            fake_pickle.return_value = no_meta_builder
            pickle.load = fake_pickle
            builder = RingBuilder.load("fake.builder", open=fake_open)
            fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
            self.assertEquals(builder.devs, rb.devs)
            fake_pickle.reset_mock()
        finally:
            pickle.load = real_pickle