Python EtcdConfiguration.get方法代码示例

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def register(node_id):
        """
        Adds a Node with a given node_id to the model
        :param node_id: ID of the ALBA node
        :type node_id: str

        :return: None
        """
        node = AlbaNodeList.get_albanode_by_node_id(node_id)
        if node is None:
            main_config = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id))
            node = AlbaNode()
            node.ip = main_config['ip']
            node.port = main_config['port']
            node.username = main_config['username']
            node.password = main_config['password']
            node.storagerouter = StorageRouterList.get_by_ip(main_config['ip'])
        data = node.client.get_metadata()
        if data['_success'] is False and data['_error'] == 'Invalid credentials':
            raise RuntimeError('Invalid credentials')
        if data['node_id'] != node_id:
            AlbaNodeController._logger.error('Unexpected node_id: {0} vs {1}'.format(data['node_id'], node_id))
            raise RuntimeError('Unexpected node identifier')
        node.node_id = node_id
        node.type = 'ASD'
        node.save()

        # increase maintenance agents count for all nodes by 1
        for backend in AlbaBackendList.get_albabackends():
            nr_of_agents_key = AlbaNodeController.NR_OF_AGENTS_ETCD_TEMPLATE.format(backend.guid)
            if EtcdConfiguration.exists(nr_of_agents_key):
                EtcdConfiguration.set(nr_of_agents_key, int(EtcdConfiguration.get(nr_of_agents_key) + 1))
            else:
                EtcdConfiguration.set(nr_of_agents_key, 1)
        AlbaNodeController.checkup_maintenance_agents()

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def pulse():
        """
        Update the heartbeats for all Storage Routers
        :return: None
        """
        logger = LogHandler.get('extensions', name='heartbeat')

        current_time = int(time.time())
        machine_id = System.get_my_machine_id()
        amqp = '{0}://{1}:{2}@{3}//'.format(EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|user'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|password'),
                                            EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id)))

        celery_path = OSManager.get_path('celery')
        worker_states = check_output("{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True)
        routers = StorageRouterList.get_storagerouters()
        for node in routers:
            if node.heartbeats is None:
                node.heartbeats = {}
            if '[email protected]{0}: OK'.format(node.name) in worker_states:
                node.heartbeats['celery'] = current_time
            if node.machine_id == machine_id:
                node.heartbeats['process'] = current_time
            else:
                try:
                    # check timeout of other nodes and clear arp cache
                    if node.heartbeats and 'process' in node.heartbeats:
                        if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT:
                            check_output("/usr/sbin/arp -d {0}".format(node.name), shell=True)
                except CalledProcessError:
                    logger.exception('Error clearing ARP cache')
            node.save()

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
def run_event_consumer():
    """
    Check whether to run the event consumer
    """
    my_ip = EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(System.get_my_machine_id()))
    for endpoint in EtcdConfiguration.get('/ovs/framework/messagequeue|endpoints'):
        if endpoint.startswith(my_ip):
            return True
    return False

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
        def _process_disk(_info, _disks, _node):
            disk = _info.get('disk')
            if disk is None:
                return
            disk_status = 'uninitialized'
            disk_status_detail = ''
            disk_alba_backend_guid = ''
            if disk['available'] is False:
                osd = _info.get('osd')
                disk_alba_state = disk['state']['state']
                if disk_alba_state == 'ok':
                    if osd is None:
                        disk_status = 'initialized'
                    elif osd['id'] is None:
                        alba_id = osd['alba_id']
                        if alba_id is None:
                            disk_status = 'available'
                        else:
                            disk_status = 'unavailable'
                            alba_backend = alba_backend_map.get(alba_id)
                            if alba_backend is not None:
                                disk_alba_backend_guid = alba_backend.guid
                    else:
                        disk_status = 'error'
                        disk_status_detail = 'communicationerror'
                        disk_alba_backend_guid = self.guid

                        for asd in _node.asds:
                            if asd.asd_id == disk['asd_id'] and asd.statistics != {}:
                                disk_status = 'warning'
                                disk_status_detail = 'recenterrors'

                                read = osd['read'] or [0]
                                write = osd['write'] or [0]
                                errors = osd['errors']
                                global_interval_key = '/ovs/alba/backends/global_gui_error_interval'
                                backend_interval_key = '/ovs/alba/backends/{0}/gui_error_interval'.format(self.guid)
                                interval = EtcdConfiguration.get(global_interval_key)
                                if EtcdConfiguration.exists(backend_interval_key):
                                    interval = EtcdConfiguration.get(backend_interval_key)
                                if len(errors) == 0 or (len(read + write) > 0 and max(min(read), min(write)) > max(error[0] for error in errors) + interval):
                                    disk_status = 'claimed'
                                    disk_status_detail = ''
                elif disk_alba_state == 'decommissioned':
                    disk_status = 'unavailable'
                    disk_status_detail = 'decommissioned'
                else:
                    disk_status = 'error'
                    disk_status_detail = disk['state']['detail']
                    alba_backend = alba_backend_map.get(osd.get('alba_id'))
                    if alba_backend is not None:
                        disk_alba_backend_guid = alba_backend.guid
            disk['status'] = disk_status
            disk['status_detail'] = disk_status_detail
            disk['alba_backend_guid'] = disk_alba_backend_guid
            _disks.append(disk)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def __init__(self, config_type, vpool_name, number=None):
        """
        Initializes the class
        """

        def make_configure(sct):
            """
            section closure
            :param sct: Section to create configure function for
            """
            return lambda **kwargs: self._add(sct, **kwargs)

        if config_type not in ['storagedriver', 'metadataserver']:
            raise RuntimeError('Invalid configuration type. Allowed: storagedriver, metadataserver')
        self.config_type = config_type
        self.vpool_name = vpool_name
        self.configuration = {}
        self.is_new = True
        self.dirty_entries = []
        self.number = number
        self.params = copy.deepcopy(StorageDriverConfiguration.parameters)  # Never use parameters directly
        self.base_path = '{0}/storagedriver/{1}'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir'), self.config_type)
        if self.number is None:
            self.path = '{0}/{1}.json'.format(self.base_path, self.vpool_name)
        else:
            self.path = '{0}/{1}_{2}.json'.format(self.base_path, self.vpool_name, self.number)
        # Fix some manual "I know what I'm doing" overrides
        backend_connection_manager = 'backend_connection_manager'
        self.params[self.config_type][backend_connection_manager]['optional'].append('s3_connection_strict_consistency')
        # Generate configure_* methods
        for section in self.params[self.config_type]:
            setattr(self, 'configure_{0}'.format(section), make_configure(section))

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def load_metadata(self):
        """
        Reads the metadata for an arakoon cluster from reality
        :return: None
        """
        key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
        if not EtcdConfiguration.exists(key):
            return

        metadata = EtcdConfiguration.get(key)
        if not isinstance(metadata, dict):
            raise ValueError('Metadata should be a dictionary')

        for key in ['in_use', 'internal', 'type']:
            if key not in metadata:
                raise ValueError('Not all required metadata keys are present for arakoon cluster {0}'.format(self.cluster_id))
            value = metadata[key]
            if key == 'in_use':
                if not isinstance(value, bool):
                    raise ValueError('"in_use" should be of type "bool"')
                self.in_use = value
            elif key == 'internal':
                if not isinstance(value, bool):
                    raise ValueError('"internal" should be of type "bool"')
                self.internal = value
            else:
                if value not in ServiceType.ARAKOON_CLUSTER_TYPES:
                    raise ValueError('Unsupported arakoon cluster type {0} found\nPlease choose from {1}'.format(value, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
                self.cluster_type = value

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def collapse_arakoon():
        """
        Collapse Arakoon's Tlogs
        :return: None
        """
        ScheduledTaskController._logger.info('Starting arakoon collapse')
        arakoon_clusters = []
        for service in ServiceList.get_services():
            if service.is_internal is True and \
               service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
                                     ServiceType.SERVICE_TYPES.NS_MGR,
                                     ServiceType.SERVICE_TYPES.ALBA_MGR):
                arakoon_clusters.append(service.name.replace('arakoon-', ''))

        for cluster in arakoon_clusters:
            ScheduledTaskController._logger.info('  Collapsing cluster {0}'.format(cluster))
            contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(cluster), raw=True)
            parser = RawConfigParser()
            parser.readfp(StringIO(contents))
            nodes = {}
            for node in parser.get('global', 'cluster').split(','):
                node = node.strip()
                nodes[node] = ([str(parser.get(node, 'ip'))], int(parser.get(node, 'client_port')))
            config = ArakoonClientConfig(str(cluster), nodes)
            for node in nodes.keys():
                ScheduledTaskController._logger.info('    Collapsing node: {0}'.format(node))
                client = ArakoonAdmin(config)
                try:
                    client.collapse(str(node), 2)
                except:
                    ScheduledTaskController._logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node))

        ScheduledTaskController._logger.info('Arakoon collapse finished')

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def collapse_arakoon():
        """
        Collapse Arakoon's Tlogs
        :return: None
        """
        logger.info('Starting arakoon collapse')
        arakoon_clusters = {}
        for service in ServiceList.get_services():
            if service.type.name in ('Arakoon', 'NamespaceManager', 'AlbaManager'):
                arakoon_clusters[service.name.replace('arakoon-', '')] = service.storagerouter

        for cluster, storagerouter in arakoon_clusters.iteritems():
            logger.info('  Collapsing cluster {0}'.format(cluster))
            contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(cluster), raw=True)
            parser = RawConfigParser()
            parser.readfp(StringIO(contents))
            nodes = {}
            for node in parser.get('global', 'cluster').split(','):
                node = node.strip()
                nodes[node] = ([parser.get(node, 'ip')], parser.get(node, 'client_port'))
            config = ArakoonClientConfig(str(cluster), nodes)
            for node in nodes.keys():
                logger.info('    Collapsing node: {0}'.format(node))
                client = ArakoonAdminClient(node, config)
                try:
                    client.collapse_tlogs(2)
                except:
                    logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node))

        logger.info('Arakoon collapse finished')

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
 def _configure_arakoon_to_volumedriver(offline_node_ips=None):
     print 'Update existing vPools'
     logger.info('Update existing vPools')
     if offline_node_ips is None:
         offline_node_ips = []
     for storagerouter in StorageRouterList.get_storagerouters():
         config = ArakoonClusterConfig('voldrv')
         config.load_config()
         arakoon_nodes = []
         for node in config.nodes:
             arakoon_nodes.append({'host': node.ip,
                                   'port': node.client_port,
                                   'node_id': node.name})
         with Remote(storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], 'ovs') as remote:
             configuration_dir = '{0}/storagedriver/storagedriver'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir'))
             if not remote.os.path.exists(configuration_dir):
                 remote.os.makedirs(configuration_dir)
             for json_file in remote.os.listdir(configuration_dir):
                 vpool_name = json_file.replace('.json', '')
                 if json_file.endswith('.json'):
                     if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)):
                         continue  # There's also a .cfg file, so this is an alba_proxy configuration file
                     storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name)
                     storagedriver_config.load()
                     storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv',
                                                                    vregistry_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon',
                                                                           dls_arakoon_cluster_id='voldrv',
                                                                           dls_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.save(reload_config=True)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
 def _configure_arakoon_to_volumedriver():
     print "Update existing vPools"
     logger.info("Update existing vPools")
     for storagerouter in StorageRouterList.get_storagerouters():
         config = ArakoonClusterConfig("voldrv")
         config.load_config()
         arakoon_nodes = []
         for node in config.nodes:
             arakoon_nodes.append({"host": node.ip, "port": node.client_port, "node_id": node.name})
         with Remote(
             storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], "ovs"
         ) as remote:
             configuration_dir = "{0}/storagedriver/storagedriver".format(
                 EtcdConfiguration.get("/ovs/framework/paths|cfgdir")
             )
             if not remote.os.path.exists(configuration_dir):
                 remote.os.makedirs(configuration_dir)
             for json_file in remote.os.listdir(configuration_dir):
                 vpool_name = json_file.replace(".json", "")
                 if json_file.endswith(".json"):
                     if remote.os.path.exists("{0}/{1}.cfg".format(configuration_dir, vpool_name)):
                         continue  # There's also a .cfg file, so this is an alba_proxy configuration file
                     storagedriver_config = remote.StorageDriverConfiguration("storagedriver", vpool_name)
                     storagedriver_config.load()
                     storagedriver_config.configure_volume_registry(
                         vregistry_arakoon_cluster_id="voldrv", vregistry_arakoon_cluster_nodes=arakoon_nodes
                     )
                     storagedriver_config.configure_distributed_lock_store(
                         dls_type="Arakoon", dls_arakoon_cluster_id="voldrv", dls_arakoon_cluster_nodes=arakoon_nodes
                     )
                     storagedriver_config.save(reload_config=True)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
 def update_status(storagedriver_id):
     """
     Sets Storage Driver offline in case hypervisor management Center
     reports the hypervisor pmachine related to this Storage Driver
     as unavailable.
     :param storagedriver_id: ID of the storagedriver to update its status
     """
     pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
     storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
     storagerouter = storagedriver.storagerouter
     if pmachine.mgmtcenter:
         # Update status
         pmachine.invalidate_dynamics(['host_status'])
     else:
         # No management Center, cannot update status via api
         logger.info('Updating status of pmachine {0} using SSHClient'.format(pmachine.name))
         host_status = 'RUNNING'
         try:
             client = SSHClient(storagerouter, username='root')
             configuration_dir = EtcdConfiguration.get('/ovs/framework/paths|cfgdir')
             logger.info('SSHClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
             with Remote(client.ip, [LocalStorageRouterClient]) as remote:
                 lsrc = remote.LocalStorageRouterClient('{0}/storagedriver/storagedriver/{1}.json'.format(configuration_dir,
                                                                                                          storagedriver.vpool.name))
                 lsrc.server_revision()
             logger.info('LocalStorageRouterClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
         except Exception as ex:
             logger.error('Connectivity check failed, assuming host {0} is halted. {1}'.format(pmachine.name, ex))
             host_status = 'HALTED'
         if host_status != 'RUNNING':
             # Host is stopped
             storagedriver_client = StorageDriverClient.load(storagedriver.vpool)
             storagedriver_client.mark_node_offline(str(storagedriver.storagedriver_id))

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def load_config(self):
        """
        Reads a configuration from reality
        """
        contents = EtcdConfiguration.get(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), raw=True)
        parser = RawConfigParser()
        parser.readfp(StringIO(contents))

        self.nodes = []
        self._extra_globals = {}
        for key in parser.options('global'):
            if key == 'plugins':
                self._plugins = [plugin.strip() for plugin in parser.get('global', 'plugins').split(',')]
            elif key in ['cluster_id', 'cluster']:
                pass  # Ignore these
            else:
                self._extra_globals[key] = parser.get('global', key)
        for node in parser.get('global', 'cluster').split(','):
            node = node.strip()
            self.nodes.append(ArakoonNodeConfig(name=node,
                                                ip=parser.get(node, 'ip'),
                                                client_port=parser.get(node, 'client_port'),
                                                messaging_port=parser.get(node, 'messaging_port'),
                                                log_dir=parser.get(node, 'log_dir'),
                                                home=parser.get(node, 'home'),
                                                tlog_dir=parser.get(node, 'tlog_dir')))

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def migrate(master_ips=None, extra_ips=None):
        """
        Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
        :param master_ips: IP addresses of the MASTER nodes
        :param extra_ips: IP addresses of the EXTRA nodes
        """

        data = EtcdConfiguration.get('/ovs/framework/versions') if EtcdConfiguration.exists('/ovs/framework/versions') else {}
        migrators = []
        path = os.path.join(os.path.dirname(__file__), 'migration')
        for filename in os.listdir(path):
            if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py'):
                name = filename.replace('.py', '')
                module = imp.load_source(name, os.path.join(path, filename))
                for member in inspect.getmembers(module):
                    if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
                        migrators.append((member[1].identifier, member[1].migrate))

        end_version = 0
        for identifier, method in migrators:
            base_version = data[identifier] if identifier in data else 0
            version = method(base_version, master_ips, extra_ips)
            if version > end_version:
                end_version = version
            data[identifier] = end_version

        EtcdConfiguration.set('/ovs/framework/versions', data)

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
 def _has_plugin(self):
     """
     Checks whether this BackendType has a plugin installed
     """
     try:
         return self.code in EtcdConfiguration.get('/ovs/framework/plugins/installed|backends')
     except:
         return False

# 需要导入模块: from ovs.extensions.db.etcd.configuration import EtcdConfiguration [as 别名]
# 或者: from ovs.extensions.db.etcd.configuration.EtcdConfiguration import get [as 别名]
    def get_mds_storagedriver_config_set(vpool, check_online=False):
        """
        Builds a configuration for all StorageRouters from a given VPool with following goals:
        * Primary MDS is the local one
        * All slaves are on different hosts
        * Maximum `mds.safety` nodes are returned
        The configuration returned is the default configuration used by the volumedriver of which in normal use-cases
        only the 1st entry is used, because at volume creation time, the volumedriver needs to create 1 master MDS
        During ensure_safety, we actually create/set the MDS slaves for each volume
        :param vpool: vPool to get storagedriver configuration for
        :param check_online: Check whether the storage routers are actually responsive
        """

        mds_per_storagerouter = {}
        mds_per_load = {}
        for storagedriver in vpool.storagedrivers:
            storagerouter = storagedriver.storagerouter
            if check_online is True:
                try:
                    client = SSHClient(storagerouter)
                    client.run('pwd')
                except UnableToConnectException:
                    continue
            mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True)
            mds_per_storagerouter[storagerouter] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]}
            if load not in mds_per_load:
                mds_per_load[load] = []
            mds_per_load[load].append(storagerouter)

        safety = EtcdConfiguration.get('/ovs/framework/storagedriver|mds_safety')
        config_set = {}
        for storagerouter, ip_info in mds_per_storagerouter.iteritems():
            primary_failure_domain = storagerouter.primary_failure_domain
            secondary_failure_domain = storagerouter.secondary_failure_domain
            config_set[storagerouter.guid] = [ip_info]
            for load in sorted(mds_per_load):
                if len(config_set[storagerouter.guid]) >= safety:
                    break
                other_storagerouters = mds_per_load[load]
                random.shuffle(other_storagerouters)
                for other_storagerouter in other_storagerouters:
                    if len(config_set[storagerouter.guid]) >= safety:
                        break
                    if other_storagerouter != storagerouter and other_storagerouter in primary_failure_domain.primary_storagerouters:
                        config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter])
            if secondary_failure_domain is not None:
                for load in sorted(mds_per_load):
                    if len(config_set[storagerouter.guid]) >= safety:
                        break
                    other_storagerouters = mds_per_load[load]
                    random.shuffle(other_storagerouters)
                    for other_storagerouter in other_storagerouters:
                        if len(config_set[storagerouter.guid]) >= safety:
                            break
                        if other_storagerouter != storagerouter and other_storagerouter in secondary_failure_domain.primary_storagerouters:
                            config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter])
        return config_set