Python EtcdConfiguration.set方法代码示例

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def migrate(master_ips=None, extra_ips=None):
        """
        Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
        :param master_ips: IP addresses of the MASTER nodes
        :param extra_ips: IP addresses of the EXTRA nodes
        """

        data = EtcdConfiguration.get('/ovs/framework/versions') if EtcdConfiguration.exists('/ovs/framework/versions') else {}
        migrators = []
        path = os.path.join(os.path.dirname(__file__), 'migration')
        for filename in os.listdir(path):
            if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py'):
                name = filename.replace('.py', '')
                module = imp.load_source(name, os.path.join(path, filename))
                for member in inspect.getmembers(module):
                    if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
                        migrators.append((member[1].identifier, member[1].migrate))

        end_version = 0
        for identifier, method in migrators:
            base_version = data[identifier] if identifier in data else 0
            version = method(base_version, master_ips, extra_ips)
            if version > end_version:
                end_version = version
            data[identifier] = end_version

        EtcdConfiguration.set('/ovs/framework/versions', data)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def save(self, client=None, reload_config=True):
     """
     Saves the configuration to a given file, optionally a remote one
     :param client: If provided, save remote configuration
     :param reload_config: Reload the running Storage Driver configuration
     """
     self._validate()
     contents = json.dumps(self.configuration, indent=4)
     EtcdConfiguration.set(self.path, contents, raw=True)
     if self.config_type == 'storagedriver' and reload_config is True:
         if len(self.dirty_entries) > 0:
             if client is None:
                 logger.info('Applying local storagedriver configuration changes')
                 changes = LSRClient(self.remote_path).update_configuration(self.remote_path)
             else:
                 logger.info('Applying storagedriver configuration changes on {0}'.format(client.ip))
                 with Remote(client.ip, [LSRClient]) as remote:
                     changes = copy.deepcopy(remote.LocalStorageRouterClient(self.remote_path).update_configuration(self.remote_path))
             for change in changes:
                 if change['param_name'] not in self.dirty_entries:
                     raise RuntimeError('Unexpected configuration change: {0}'.format(change['param_name']))
                 logger.info('Changed {0} from "{1}" to "{2}"'.format(change['param_name'], change['old_value'], change['new_value']))
                 self.dirty_entries.remove(change['param_name'])
             logger.info('Changes applied')
             if len(self.dirty_entries) > 0:
                 logger.warning('Following changes were not applied: {0}'.format(', '.join(self.dirty_entries)))
         else:
             logger.debug('No need to apply changes, nothing changed')
     self.is_new = False
     self.dirty_entries = []

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def register(node_id):
        """
        Adds a Node with a given node_id to the model
        :param node_id: ID of the ALBA node
        :type node_id: str

        :return: None
        """
        node = AlbaNodeList.get_albanode_by_node_id(node_id)
        if node is None:
            main_config = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id))
            node = AlbaNode()
            node.ip = main_config['ip']
            node.port = main_config['port']
            node.username = main_config['username']
            node.password = main_config['password']
            node.storagerouter = StorageRouterList.get_by_ip(main_config['ip'])
        data = node.client.get_metadata()
        if data['_success'] is False and data['_error'] == 'Invalid credentials':
            raise RuntimeError('Invalid credentials')
        if data['node_id'] != node_id:
            AlbaNodeController._logger.error('Unexpected node_id: {0} vs {1}'.format(data['node_id'], node_id))
            raise RuntimeError('Unexpected node identifier')
        node.node_id = node_id
        node.type = 'ASD'
        node.save()

        # increase maintenance agents count for all nodes by 1
        for backend in AlbaBackendList.get_albabackends():
            nr_of_agents_key = AlbaNodeController.NR_OF_AGENTS_ETCD_TEMPLATE.format(backend.guid)
            if EtcdConfiguration.exists(nr_of_agents_key):
                EtcdConfiguration.set(nr_of_agents_key, int(EtcdConfiguration.get(nr_of_agents_key) + 1))
            else:
                EtcdConfiguration.set(nr_of_agents_key, 1)
        AlbaNodeController.checkup_maintenance_agents()

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def get_path(binary_name):
     machine_id = System.get_my_machine_id()
     config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(machine_id, binary_name)
     path = EtcdConfiguration.get(config_location)
     if not path:
         try:
             path = check_output('which {0}'.format(binary_name), shell=True).strip()
             EtcdConfiguration.set(config_location, path)
         except CalledProcessError:
             return None
     return path

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def write(self):
        """
        Write the metadata to Etcd
        :return: None
        """
        if self.cluster_type is None or self.cluster_type == '':
            raise ValueError('Cluster type must be defined before being able to store the cluster metadata information')

        etcd_key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
        EtcdConfiguration.set(key=etcd_key, value={'type': self.cluster_type,
                                                   'in_use': self.in_use,
                                                   'internal': self.internal})

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def ovs_3977_maintenance_agent_test():
        """
        Test maintenance agent processes
        """
        def _get_agent_distribution(agent_name):
            result = {}
            total = 0
            for ip in alba_node_ips:
                count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
                if count:
                    count = int(count)
                else:
                    count = 0
                total += count
                result[ip] = count
            result['total'] = total

            print 'Maintenance agent distribution: {0}'.format(result)
            for ip in alba_node_ips:
                assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
                    "Agents not equally distributed!"

            return result

        backend = GeneralBackend.get_by_name(TestALBA.backend_name)
        if backend is None:
            backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
        name = backend.alba_backend.name

        alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]

        etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
        nr_of_agents = EtcdConfiguration.get(etcd_key)
        print '1. - nr of agents: {0}'.format(nr_of_agents)

        actual_nr_of_agents = _get_agent_distribution(name)['total']
        assert nr_of_agents == actual_nr_of_agents, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)

        # set nr to zero
        EtcdConfiguration.set(etcd_key, 0)
        GeneralAlba.checkup_maintenance_agents()
        assert _get_agent_distribution(name)['total'] == 0, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
        print '2. - nr of agents: {0}'.format(nr_of_agents)

        # set nr to 10
        EtcdConfiguration.set(etcd_key, 10)
        GeneralAlba.checkup_maintenance_agents()
        assert _get_agent_distribution(name)['total'] == 10, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
        print '3. - nr of agents: {0}'.format(nr_of_agents)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def write_config(self):
     """
     Writes the configuration down to in the format expected by Arakoon
     """
     contents = RawConfigParser()
     data = self.export()
     for section in data:
         contents.add_section(section)
         for item in data[section]:
             contents.set(section, item, data[section][item])
     config_io = StringIO()
     contents.write(config_io)
     EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), config_io.getvalue(), raw=True)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def write_config(self):
     """
     Writes the configuration down to in the format expected by Arakoon
     """
     (temp_handle, temp_filename) = tempfile.mkstemp()
     contents = RawConfigParser()
     data = self.export()
     for section in data:
         contents.add_section(section)
         for item in data[section]:
             contents.set(section, item, data[section][item])
     with open(temp_filename, 'wb') as config_file:
         contents.write(config_file)
     with open(temp_filename, 'r') as the_file:
         EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), the_file.read(), raw=True)
     os.remove(temp_filename)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def register(name, email, company, phone, newsletter):
     """
     Registers the environment
     """
     SupportAgent().run()  # Execute a single heartbeat run
     client = OVSClient('monitoring.openvstorage.com', 443, credentials=None, verify=True, version=1)
     task_id = client.post('/support/register/',
                           data={'cluster_id': EtcdConfiguration.get('/ovs/framework/cluster_id'),
                                 'name': name,
                                 'email': email,
                                 'company': company,
                                 'phone': phone,
                                 'newsletter': newsletter,
                                 'register_only': True})
     if task_id:
         client.wait_for_task(task_id, timeout=120)
     EtcdConfiguration.set('/ovs/framework/registered', True)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
 def get_path(binary_name):
     """
     Retrieve the absolute path for binary
     :param binary_name: Binary to get path for
     :return: Path
     """
     machine_id = System.get_my_machine_id()
     config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(machine_id, binary_name)
     if not EtcdConfiguration.exists(config_location):
         try:
             path = check_output('which {0}'.format(binary_name), shell=True).strip()
             EtcdConfiguration.set(config_location, path)
         except CalledProcessError:
             return None
     else:
         path = EtcdConfiguration.get(config_location)
     return path

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def verify_namespaces():
        """
        Verify namespaces for all backends
        """
        logger.info('verify namespace task scheduling started')

        job_factor = 10
        job_factor_key = '/ovs/alba/backends/job_factor'
        if EtcdConfiguration.exists(job_factor_key):
            job_factor = EtcdConfiguration.get(job_factor_key)
        else:
            EtcdConfiguration.set(job_factor_key, job_factor)

        for albabackend in AlbaBackendList.get_albabackends():
            config = 'etcd://127.0.0.1:2379/ovs/arakoon/{0}-abm/config'.format(albabackend.backend.name)
            namespaces = AlbaCLI.run('list-namespaces', config=config, as_json=True)
            for namespace in namespaces:
                logger.info('verifying namespace: {0} scheduled ...'.format(namespace['name']))
                AlbaCLI.run('verify-namespace {0} --factor={1}'.format(namespace['name'], job_factor))

        logger.info('verify namespace task scheduling finished')

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def verify_namespaces():
        """
        Verify namespaces for all backends
        """
        AlbaScheduledTaskController._logger.info('verify namespace task scheduling started')

        verification_factor = 10
        verification_factor_key = '/ovs/alba/backends/verification_factor'
        if EtcdConfiguration.exists(verification_factor_key):
            verification_factor = EtcdConfiguration.get(verification_factor_key)
        else:
            EtcdConfiguration.set(verification_factor_key, verification_factor)

        for albabackend in AlbaBackendList.get_albabackends():
            backend_name = albabackend.abm_services[0].service.name if albabackend.abm_services else albabackend.name + '-abm'
            config = 'etcd://127.0.0.1:2379/ovs/arakoon/{0}/config'.format(backend_name)
            namespaces = AlbaCLI.run('list-namespaces', config=config, as_json=True)
            for namespace in namespaces:
                AlbaScheduledTaskController._logger.info('verifying namespace: {0} scheduled ...'.format(namespace['name']))
                AlbaCLI.run('verify-namespace {0} --factor={1}'.format(namespace['name'], verification_factor))

        AlbaScheduledTaskController._logger.info('verify namespace task scheduling finished')

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def migrate(master_ips=None, extra_ips=None):
        """
        Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
        :param master_ips: IP addresses of the MASTER nodes
        :param extra_ips: IP addresses of the EXTRA nodes
        """
        machine_id = System.get_my_machine_id()
        key = '/ovs/framework/hosts/{0}/versions'.format(machine_id)
        try:
            data = EtcdConfiguration.get(key) if EtcdConfiguration.exists(key) else {}
        except EtcdConnectionFailed:
            import json  # Most likely 2.6 to 2.7 migration
            data = {}
            filename = '/opt/OpenvStorage/config/ovs.json'
            if os.path.exists(filename):
                with open(filename) as config_file:
                    data = json.load(config_file).get('core', {}).get('versions', {})
        migrators = []
        path = os.path.join(os.path.dirname(__file__), 'migration')
        for filename in os.listdir(path):
            if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py'):
                name = filename.replace('.py', '')
                module = imp.load_source(name, os.path.join(path, filename))
                for member in inspect.getmembers(module):
                    if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
                        migrators.append((member[1].identifier, member[1].migrate))

        end_version = 0
        for identifier, method in migrators:
            base_version = data[identifier] if identifier in data else 0
            version = method(base_version, master_ips, extra_ips)
            if version > end_version:
                end_version = version
            data[identifier] = end_version

        EtcdConfiguration.set(key, data)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def checkup_maintenance_agents():
        """
        Check if requested nr of maintenance agents / backend is actually present
        Add / remove as necessary
        :return: None
        """
        service_template_key = 'alba-maintenance_{0}-{1}'
        maintenance_agents_map = {}
        asd_nodes = AlbaNodeList.get_albanodes()
        nr_of_storage_nodes = len(asd_nodes)

        def _get_node_load(backend_name):
            highest_load = 0
            lowest_load = sys.maxint
            agent_load = {'high_load_node': asd_nodes[0] if asd_nodes else None,
                          'low_load_node': asd_nodes[0] if asd_nodes else None,
                          'total_load': 0}
            for asd_node in asd_nodes:
                actual_nr_of_agents = 0
                maint_services = asd_node.client.list_maintenance_services()
                for service_name in maint_services:
                    if service_template_key.format(backend_name, '') in service_name:
                        actual_nr_of_agents += 1
                if actual_nr_of_agents > highest_load:
                    agent_load['high_load_node'] = asd_node
                    highest_load = actual_nr_of_agents
                if actual_nr_of_agents < lowest_load:
                    agent_load['low_load_node'] = asd_node
                    lowest_load = actual_nr_of_agents
                agent_load['total_load'] += actual_nr_of_agents

            return agent_load

        alba_backends = AlbaBackendList.get_albabackends()
        for alba_backend in alba_backends:
            nr_of_agents_key = AlbaNodeController.NR_OF_AGENTS_ETCD_TEMPLATE.format(alba_backend.guid)
            name = alba_backend.backend.name
            if not EtcdConfiguration.exists(nr_of_agents_key):
                EtcdConfiguration.set(nr_of_agents_key, nr_of_storage_nodes)
            required_nr = EtcdConfiguration.get(nr_of_agents_key)
            maintenance_agents_map[name] = {'required': required_nr,
                                            'actual': _get_node_load(name)['total_load'],
                                            'backend': alba_backend.backend}

        for name, values in maintenance_agents_map.iteritems():
            AlbaNodeController._logger.info('Checking backend: {0}'.format(name))
            to_process = values['required'] - values['actual']

            if to_process == 0:
                AlbaNodeController._logger.info('No action required for: {0}'.format(name))
            elif to_process >= 0:
                AlbaNodeController._logger.info('Adding {0} maintenance agent(s) for {1}'.format(to_process, name))
                for _ in xrange(to_process):
                    unique_hash = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(16))
                    node = _get_node_load(name)['low_load_node']
                    AlbaNodeController._logger.info('Service to add: ' + service_template_key.format(name, unique_hash))
                    if node and node.client:
                        node.client.add_maintenance_service(service_template_key.format(name, unique_hash),
                                                            values['backend'].alba_backend.guid,
                                                            AlbaController.get_abm_service_name(values['backend']))
                        AlbaNodeController._logger.info('Service added')
            else:
                to_process = abs(to_process)
                AlbaNodeController._logger.info('Removing {0} maintenance agent(s) for {1}'.format(to_process, name))
                for _ in xrange(to_process):
                    node = _get_node_load(name)['high_load_node']
                    services = node.client.list_maintenance_services()
                    if services and node and node.client:
                        for service in services:
                            if 'alba-maintenance_' + name in service:
                                node.client.remove_maintenance_service(service)
                                break

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import set [as 别名]
    def _voldrv_arakoon_checkup(create_cluster):
        def add_service(service_storagerouter, arakoon_ports):
            """
            Add a service to the storage router
            :param service_storagerouter: Storage Router to add the service to
            :param arakoon_ports: Port information
            :return: The newly created and added service
            """
            new_service = Service()
            new_service.name = service_name
            new_service.type = service_type
            new_service.ports = arakoon_ports
            new_service.storagerouter = service_storagerouter
            new_service.save()
            return new_service

        service_name = 'arakoon-voldrv'
        service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)

        current_ips = []
        current_services = []
        for service in service_type.services:
            if service.name == service_name:
                current_services.append(service)
                if service.is_internal is True:
                    current_ips.append(service.storagerouter.ip)

        all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
        available_storagerouters = {}
        for storagerouter in StorageRouterList.get_masters():
            storagerouter.invalidate_dynamics(['partition_config'])
            if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
                available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
            all_sr_ips.append(storagerouter.ip)

        if create_cluster is True and len(current_services) == 0:  # Create new cluster
            metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
            if metadata is None:  # No externally managed cluster found, we create 1 ourselves
                if not available_storagerouters:
                    raise RuntimeError('Could not find any Storage Router with a DB role')

                storagerouter, partition = available_storagerouters.items()[0]
                result = ArakoonInstaller.create_cluster(cluster_name='voldrv',
                                                         cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
                                                         ip=storagerouter.ip,
                                                         base_dir=partition.folder,
                                                         claim=True)
                ports = [result['client_port'], result['messaging_port']]
                metadata = result['metadata']
                ArakoonInstaller.restart_cluster_add(cluster_name='voldrv', current_ips=current_ips, new_ip=storagerouter.ip)
                current_ips.append(storagerouter.ip)
            else:
                ports = []
                storagerouter = None

            cluster_name = metadata.cluster_id
            EtcdConfiguration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name)
            StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name))
            StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
            current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports))

        cluster_name = EtcdConfiguration.get('/ovs/framework/arakoon_clusters').get('voldrv')
        if cluster_name is None:
            return
        metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
        if 0 < len(current_services) < len(available_storagerouters) and metadata.internal is True:
            for storagerouter, partition in available_storagerouters.iteritems():
                if storagerouter.ip in current_ips:
                    continue
                result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip,
                                                         new_ip=storagerouter.ip,
                                                         cluster_name=cluster_name,
                                                         base_dir=partition.folder)
                add_service(storagerouter, [result['client_port'], result['messaging_port']])
                current_ips.append(storagerouter.ip)
                ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
            StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)