Python storagerouterlist.StorageRouterList类代码示例

    def _dtl_status(self):
        """
        Retrieve the DTL status for a vDisk
        """
        sd_status = self.info.get('failover_mode', 'UNKNOWN').lower()
        if sd_status == '':
            sd_status = 'unknown'
        if sd_status != 'ok_standalone':
            return sd_status

        # Verify whether 'ok_standalone' is the correct status for this vDisk
        vpool_dtl = self.vpool.configuration['dtl_enabled']
        if self.has_manual_dtl is True or vpool_dtl is False:
            return sd_status

        domains = []
        possible_dtl_targets = set()
        for sr in StorageRouterList.get_storagerouters():
            if sr.guid == self.storagerouter_guid:
                domains = [junction.domain for junction in sr.domains]
            elif len(sr.storagedrivers) > 0:
                possible_dtl_targets.add(sr)

        if len(domains) > 0:
            possible_dtl_targets = set()
            for domain in domains:
                possible_dtl_targets.update(StorageRouterList.get_primary_storagerouters_for_domain(domain))

        if len(possible_dtl_targets) == 0:
            return sd_status
        return 'checkup_required'

    def restart_framework_and_memcache_services(clients, logger, offline_node_ips=None):
        """
        Restart framework and Memcached services
        :param clients: Clients on which to restart these services
        :type clients: dict
        :param logger: Logger object used for logging
        :type logger: ovs.log.log_handler.LogHandler
        :param offline_node_ips: IP addresses of offline nodes in the cluster
        :type offline_node_ips: list
        :return: None
        """
        from ovs.dal.lists.storagerouterlist import StorageRouterList

        master_ips = [sr.ip for sr in StorageRouterList.get_masters()]
        slave_ips = [sr.ip for sr in StorageRouterList.get_slaves()]
        if offline_node_ips is None:
            offline_node_ips = []
        memcached = 'memcached'
        watcher = 'watcher-framework'
        support_agent = 'support-agent'
        for ip in master_ips + slave_ips:
            if ip not in offline_node_ips:
                if ServiceManager.has_service(watcher, clients[ip]):
                    Toolbox.change_service_state(clients[ip], watcher, 'stop', logger)
        for ip in master_ips:
            if ip not in offline_node_ips:
                Toolbox.change_service_state(clients[ip], memcached, 'restart', logger)
        for ip in master_ips + slave_ips:
            if ip not in offline_node_ips:
                if ServiceManager.has_service(watcher, clients[ip]):
                    Toolbox.change_service_state(clients[ip], watcher, 'start', logger)
                if ServiceManager.has_service(support_agent, clients[ip]):
                    Toolbox.change_service_state(clients[ip], support_agent, 'restart', logger)
        VolatileFactory.store = None

    def _voldrv_arakoon_checkup(create_cluster):
        def add_service(service_storagerouter, arakoon_result):
            """
            Add a service to the storage router
            :param service_storagerouter: Storage Router to add the service to
            :type service_storagerouter:  StorageRouter

            :param arakoon_result:        Port information
            :type arakoon_result:         Dictionary

            :return:                      The newly created and added service
            """
            new_service = Service()
            new_service.name = service_name
            new_service.type = service_type
            new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']]
            new_service.storagerouter = service_storagerouter
            new_service.save()
            return new_service

        cluster_name = 'voldrv'
        service_name = 'arakoon-voldrv'
        service_type = ServiceTypeList.get_by_name('Arakoon')
        current_services = []
        current_ips = []
        for service in service_type.services:
            if service.name == service_name:
                current_services.append(service)
                current_ips.append(service.storagerouter.ip)
        all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
        available_storagerouters = {}
        for storagerouter in StorageRouterList.get_masters():
            storagerouter.invalidate_dynamics(['partition_config'])
            if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
                available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
            all_sr_ips.append(storagerouter.ip)
        if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0:
            storagerouter, partition = available_storagerouters.items()[0]
            result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
                                                     ip=storagerouter.ip,
                                                     base_dir=partition.folder)
            current_services.append(add_service(storagerouter, result))
            ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
            current_ips.append(storagerouter.ip)
            StorageDriverController._configure_arakoon_to_volumedriver()

        if 0 < len(current_services) < len(available_storagerouters):
            for storagerouter, partition in available_storagerouters.iteritems():
                if storagerouter.ip in current_ips:
                    continue
                result = ArakoonInstaller.extend_cluster(
                    current_services[0].storagerouter.ip,
                    storagerouter.ip,
                    cluster_name,
                    partition.folder
                )
                add_service(storagerouter, result)
                current_ips.append(storagerouter.ip)
                ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
            StorageDriverController._configure_arakoon_to_volumedriver()

 def print_current_mds_layout():
     """
     Prints the current MDS layout
     """
     output = ['',
               'Open vStorage - MDS debug information',
               '=====================================',
               'timestamp: {0}'.format(time.time()),
               '']
     for storagerouter in StorageRouterList.get_storagerouters():
         output.append('+ {0} ({1})'.format(storagerouter.name, storagerouter.ip))
         vpools = set(sd.vpool for sd in storagerouter.storagedrivers)
         for vpool in vpools:
             output.append('  + {0}'.format(vpool.name))
             for mds_service in vpool.mds_services:
                 if mds_service.service.storagerouter_guid == storagerouter.guid:
                     masters, slaves = 0, 0
                     for junction in mds_service.vdisks:
                         if junction.is_master:
                             masters += 1
                         else:
                             slaves += 1
                     capacity = mds_service.capacity
                     if capacity == -1:
                         capacity = 'infinite'
                     load, _ = MDSServiceController.get_mds_load(mds_service)
                     if load == float('inf'):
                         load = 'infinite'
                     else:
                         load = '{0}%'.format(round(load, 2))
                     output.append('    + {0} - port {1} - {2} master(s), {3} slave(s) - capacity: {4}, load: {5}'.format(
                         mds_service.number, mds_service.service.ports[0], masters, slaves, capacity, load
                     ))
     print '\n'.join(output)

    def pulse():
        """
        Update the heartbeats for the Current Routers
        :return: None
        """
        logger = LogHandler.get('extensions', name='heartbeat')
        machine_id = System.get_my_machine_id()
        current_time = int(time.time())

        routers = StorageRouterList.get_storagerouters()
        for node in routers:
            if node.machine_id == machine_id:
                with volatile_mutex('storagerouter_heartbeat_{0}'.format(node.guid)):
                    node_save = StorageRouter(node.guid)
                    node_save.heartbeats['process'] = current_time
                    node_save.save()
                StorageRouterController.ping.s(node.guid, current_time).apply_async(routing_key='sr.{0}'.format(machine_id))
            else:
                try:
                    # check timeout of other nodes and clear arp cache
                    if node.heartbeats and 'process' in node.heartbeats:
                        if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT:
                            check_output("/usr/sbin/arp -d '{0}'".format(node.name.replace(r"'", r"'\''")), shell=True)
                except CalledProcessError:
                    logger.exception('Error clearing ARP cache')

 def get_storage_router_by_ip(ip):
     """
     Retrieve Storage Router based on IP
     :param ip: IP of Storage Router
     :return: Storage Router DAL object
     """
     return StorageRouterList.get_by_ip(ip)

    def pulse():
        """
        Update the heartbeats for all Storage Routers
        :return: None
        """
        logger = LogHandler.get('extensions', name='heartbeat')

        current_time = int(time.time())
        machine_id = System.get_my_machine_id()
        amqp = '{0}://{1}:{2}@{3}//'.format(EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|user'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|password'),
                                            EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id)))

        celery_path = OSManager.get_path('celery')
        worker_states = check_output("{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True)
        routers = StorageRouterList.get_storagerouters()
        for node in routers:
            if node.heartbeats is None:
                node.heartbeats = {}
            if '[email protected]{0}: OK'.format(node.name) in worker_states:
                node.heartbeats['celery'] = current_time
            if node.machine_id == machine_id:
                node.heartbeats['process'] = current_time
            else:
                try:
                    # check timeout of other nodes and clear arp cache
                    if node.heartbeats and 'process' in node.heartbeats:
                        if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT:
                            check_output("/usr/sbin/arp -d {0}".format(node.name), shell=True)
                except CalledProcessError:
                    logger.exception('Error clearing ARP cache')
            node.save()

    def _run_and_validate_dtl_checkup(self, vdisk, validations):
        """
        Execute the DTL checkup for a vDisk and validate the settings afterwards
        """
        single_node = len(StorageRouterList.get_storagerouters()) == 1
        VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
        config = vdisk.storagedriver_client.get_dtl_config(vdisk.volume_id)
        config_mode = vdisk.storagedriver_client.get_dtl_config_mode(vdisk.volume_id)
        msg = '{0} node - {{0}} - Actual: {{1}} - Expected: {{2}}'.format('Single' if single_node is True else 'Multi')

        validations.append({'key': 'config_mode', 'value': DTLConfigMode.MANUAL})
        for validation in validations:
            key = validation['key']
            value = validation['value']
            if key == 'config':
                actual_value = config
            elif key == 'host':
                actual_value = config.host
            elif key == 'port':
                actual_value = config.port
            elif key == 'mode':
                actual_value = config.mode
            else:
                actual_value = config_mode

            if isinstance(value, list):
                self.assertTrue(expr=actual_value in value,
                                msg=msg.format(key.capitalize(), actual_value, ', '.join(value)))
            else:
                self.assertEqual(first=actual_value,
                                 second=value,
                                 msg=msg.format(key.capitalize(), actual_value, value))
        return config

 def _configure_arakoon_to_volumedriver():
     print 'Update existing vPools'
     logger.info('Update existing vPools')
     for storagerouter in StorageRouterList.get_storagerouters():
         with Remote(storagerouter.ip, [os, RawConfigParser, Configuration, StorageDriverConfiguration, ArakoonManagementEx], 'ovs') as remote:
             arakoon_cluster_config = remote.ArakoonManagementEx().getCluster('voldrv').getClientConfig()
             arakoon_nodes = []
             for node_id, node_config in arakoon_cluster_config.iteritems():
                 arakoon_nodes.append({'host': node_config[0][0],
                                       'port': node_config[1],
                                       'node_id': node_id})
             configuration_dir = '{0}/storagedriver/storagedriver'.format(
                 remote.Configuration.get('ovs.core.cfgdir'))
             if not remote.os.path.exists(configuration_dir):
                 remote.os.makedirs(configuration_dir)
             for json_file in remote.os.listdir(configuration_dir):
                 vpool_name = json_file.replace('.json', '')
                 if json_file.endswith('.json'):
                     if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)):
                         continue  # There's also a .cfg file, so this is an alba_proxy configuration file
                     storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name)
                     storagedriver_config.load()
                     storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv',
                                                                    vregistry_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon',
                                                                           dls_arakoon_cluster_id='voldrv',
                                                                           dls_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.save()

 def on_demote(cluster_ip, master_ip):
     """
     Handles the demote for the StorageDrivers
     :param cluster_ip: IP of the node to demote
     :param master_ip: IP of the master node
     """
     client = SSHClient(cluster_ip, username='root')
     servicetype = ServiceTypeList.get_by_name('Arakoon')
     current_service = None
     remaining_ips = []
     for service in servicetype.services:
         if service.name == 'arakoon-voldrv':
             if service.storagerouter.ip == cluster_ip:
                 current_service = service
             else:
                 remaining_ips.append(service.storagerouter.ip)
     if current_service is not None:
         print '* Shrink StorageDriver cluster'
         ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv')
         if ServiceManager.has_service(current_service.name, client=client) is True:
             ServiceManager.stop_service(current_service.name, client=client)
             ServiceManager.remove_service(current_service.name, client=client)
         ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
         current_service.delete()
         for storagerouter in StorageRouterList.get_storagerouters():
             ArakoonInstaller.deploy_to_slave(master_ip, storagerouter.ip, 'voldrv')
         StorageDriverController._configure_arakoon_to_volumedriver()

 def _configure_arakoon_to_volumedriver(offline_node_ips=None):
     print 'Update existing vPools'
     logger.info('Update existing vPools')
     if offline_node_ips is None:
         offline_node_ips = []
     for storagerouter in StorageRouterList.get_storagerouters():
         config = ArakoonClusterConfig('voldrv')
         config.load_config()
         arakoon_nodes = []
         for node in config.nodes:
             arakoon_nodes.append({'host': node.ip,
                                   'port': node.client_port,
                                   'node_id': node.name})
         with Remote(storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], 'ovs') as remote:
             configuration_dir = '{0}/storagedriver/storagedriver'.format(EtcdConfiguration.get('/ovs/framework/paths|cfgdir'))
             if not remote.os.path.exists(configuration_dir):
                 remote.os.makedirs(configuration_dir)
             for json_file in remote.os.listdir(configuration_dir):
                 vpool_name = json_file.replace('.json', '')
                 if json_file.endswith('.json'):
                     if remote.os.path.exists('{0}/{1}.cfg'.format(configuration_dir, vpool_name)):
                         continue  # There's also a .cfg file, so this is an alba_proxy configuration file
                     storagedriver_config = remote.StorageDriverConfiguration('storagedriver', vpool_name)
                     storagedriver_config.load()
                     storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id='voldrv',
                                                                    vregistry_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon',
                                                                           dls_arakoon_cluster_id='voldrv',
                                                                           dls_arakoon_cluster_nodes=arakoon_nodes)
                     storagedriver_config.save(reload_config=True)

 def register(node_id):
     """
     Adds a Node with a given node_id to the model
     :param node_id: ID of the ALBA node
     :type node_id: str
     :return: None
     """
     node = AlbaNodeList.get_albanode_by_node_id(node_id)
     if node is None:
         main_config = Configuration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id))
         node = AlbaNode()
         node.ip = main_config['ip']
         node.port = main_config['port']
         node.username = main_config['username']
         node.password = main_config['password']
         node.storagerouter = StorageRouterList.get_by_ip(main_config['ip'])
     data = node.client.get_metadata()
     if data['_success'] is False and data['_error'] == 'Invalid credentials':
         raise RuntimeError('Invalid credentials')
     if data['node_id'] != node_id:
         AlbaNodeController._logger.error('Unexpected node_id: {0} vs {1}'.format(data['node_id'], node_id))
         raise RuntimeError('Unexpected node identifier')
     node.node_id = node_id
     node.type = 'ASD'
     node.save()
     AlbaController.checkup_maintenance_agents.delay()

    def register(node_id):
        """
        Adds a Node with a given node_id to the model
        :param node_id: ID of the ALBA node
        :type node_id: str

        :return: None
        """
        node = AlbaNodeList.get_albanode_by_node_id(node_id)
        if node is None:
            main_config = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id))
            node = AlbaNode()
            node.ip = main_config['ip']
            node.port = main_config['port']
            node.username = main_config['username']
            node.password = main_config['password']
            node.storagerouter = StorageRouterList.get_by_ip(main_config['ip'])
        data = node.client.get_metadata()
        if data['_success'] is False and data['_error'] == 'Invalid credentials':
            raise RuntimeError('Invalid credentials')
        if data['node_id'] != node_id:
            AlbaNodeController._logger.error('Unexpected node_id: {0} vs {1}'.format(data['node_id'], node_id))
            raise RuntimeError('Unexpected node identifier')
        node.node_id = node_id
        node.type = 'ASD'
        node.save()

        # increase maintenance agents count for all nodes by 1
        for backend in AlbaBackendList.get_albabackends():
            nr_of_agents_key = AlbaNodeController.NR_OF_AGENTS_ETCD_TEMPLATE.format(backend.guid)
            if EtcdConfiguration.exists(nr_of_agents_key):
                EtcdConfiguration.set(nr_of_agents_key, int(EtcdConfiguration.get(nr_of_agents_key) + 1))
            else:
                EtcdConfiguration.set(nr_of_agents_key, 1)
        AlbaNodeController.checkup_maintenance_agents()

 def _configure_arakoon_to_volumedriver():
     print "Update existing vPools"
     logger.info("Update existing vPools")
     for storagerouter in StorageRouterList.get_storagerouters():
         config = ArakoonClusterConfig("voldrv")
         config.load_config()
         arakoon_nodes = []
         for node in config.nodes:
             arakoon_nodes.append({"host": node.ip, "port": node.client_port, "node_id": node.name})
         with Remote(
             storagerouter.ip, [os, RawConfigParser, EtcdConfiguration, StorageDriverConfiguration], "ovs"
         ) as remote:
             configuration_dir = "{0}/storagedriver/storagedriver".format(
                 EtcdConfiguration.get("/ovs/framework/paths|cfgdir")
             )
             if not remote.os.path.exists(configuration_dir):
                 remote.os.makedirs(configuration_dir)
             for json_file in remote.os.listdir(configuration_dir):
                 vpool_name = json_file.replace(".json", "")
                 if json_file.endswith(".json"):
                     if remote.os.path.exists("{0}/{1}.cfg".format(configuration_dir, vpool_name)):
                         continue  # There's also a .cfg file, so this is an alba_proxy configuration file
                     storagedriver_config = remote.StorageDriverConfiguration("storagedriver", vpool_name)
                     storagedriver_config.load()
                     storagedriver_config.configure_volume_registry(
                         vregistry_arakoon_cluster_id="voldrv", vregistry_arakoon_cluster_nodes=arakoon_nodes
                     )
                     storagedriver_config.configure_distributed_lock_store(
                         dls_type="Arakoon", dls_arakoon_cluster_id="voldrv", dls_arakoon_cluster_nodes=arakoon_nodes
                     )
                     storagedriver_config.save(reload_config=True)

 def remove(license_guid):
     """
     Removes a license
     """
     clients = {}
     storagerouters = StorageRouterList.get_storagerouters()
     try:
         for storagerouter in storagerouters:
             clients[storagerouter] = SSHClient(storagerouter.ip)
     except UnableToConnectException:
         raise RuntimeError('Not all StorageRouters are reachable')
     lic = License(license_guid)
     if lic.can_remove is True:
         remove_functions = Toolbox.fetch_hooks('license', '{0}.remove'.format(lic.component))
         result = remove_functions[0](component=lic.component, data=lic.data, valid_until=lic.valid_until, signature=lic.signature)
         if result is True:
             lic.delete()
             license_contents = []
             for lic in LicenseList.get_licenses():
                 license_contents.append(lic.hash)
             for storagerouter in storagerouters:
                 client = clients[storagerouter]
                 client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents)))
         return result
     return None