Python configuration.EtcdConfiguration类代码示例

    def pulse():
        """
        Update the heartbeats for all Storage Routers
        :return: None
        """
        logger = LogHandler.get('extensions', name='heartbeat')

        current_time = int(time.time())
        machine_id = System.get_my_machine_id()
        amqp = '{0}://{1}:{2}@{3}//'.format(EtcdConfiguration.get('/ovs/framework/messagequeue|protocol'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|user'),
                                            EtcdConfiguration.get('/ovs/framework/messagequeue|password'),
                                            EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id)))

        celery_path = OSManager.get_path('celery')
        worker_states = check_output("{0} inspect ping -b {1} --timeout=5 2> /dev/null | grep OK | perl -pe 's/\x1b\[[0-9;]*m//g' || true".format(celery_path, amqp), shell=True)
        routers = StorageRouterList.get_storagerouters()
        for node in routers:
            if node.heartbeats is None:
                node.heartbeats = {}
            if '[email protected]{0}: OK'.format(node.name) in worker_states:
                node.heartbeats['celery'] = current_time
            if node.machine_id == machine_id:
                node.heartbeats['process'] = current_time
            else:
                try:
                    # check timeout of other nodes and clear arp cache
                    if node.heartbeats and 'process' in node.heartbeats:
                        if current_time - node.heartbeats['process'] >= HeartBeat.ARP_TIMEOUT:
                            check_output("/usr/sbin/arp -d {0}".format(node.name), shell=True)
                except CalledProcessError:
                    logger.exception('Error clearing ARP cache')
            node.save()

    def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True):
        """
        Retrieve arakoon cluster information based on its type
        :param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES)
        :type cluster_type: str

        :param locked: Execute this in a locked context
        :type locked: bool

        :return: List of ArakoonClusterMetadata objects
        :rtype: ArakoonClusterMetadata
        """
        cluster_type = cluster_type.upper()
        if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES:
            raise ValueError('Unsupported arakoon cluster type provided. Please choose from {0}'.format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
        if not EtcdConfiguration.dir_exists('/ovs/arakoon'):
            return None

        mutex = volatile_mutex('claim_arakoon_metadata', wait=10)
        try:
            if locked is True:
                mutex.acquire()

            for cluster_name in EtcdConfiguration.list('/ovs/arakoon'):
                metadata = ArakoonClusterMetadata(cluster_id=cluster_name)
                metadata.load_metadata()
                if metadata.cluster_type == cluster_type and metadata.in_use is False and metadata.internal is False:
                    metadata.claim()
                    return metadata
        finally:
            if locked is True:
                mutex.release()

 def delete_config(self):
     """
     Deletes a configuration file
     """
     key = ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id)
     if EtcdConfiguration.exists(key, raw=True):
         EtcdConfiguration.delete(key, raw=True)

    def load_metadata(self):
        """
        Reads the metadata for an arakoon cluster from reality
        :return: None
        """
        key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
        if not EtcdConfiguration.exists(key):
            return

        metadata = EtcdConfiguration.get(key)
        if not isinstance(metadata, dict):
            raise ValueError('Metadata should be a dictionary')

        for key in ['in_use', 'internal', 'type']:
            if key not in metadata:
                raise ValueError('Not all required metadata keys are present for arakoon cluster {0}'.format(self.cluster_id))
            value = metadata[key]
            if key == 'in_use':
                if not isinstance(value, bool):
                    raise ValueError('"in_use" should be of type "bool"')
                self.in_use = value
            elif key == 'internal':
                if not isinstance(value, bool):
                    raise ValueError('"internal" should be of type "bool"')
                self.internal = value
            else:
                if value not in ServiceType.ARAKOON_CLUSTER_TYPES:
                    raise ValueError('Unsupported arakoon cluster type {0} found\nPlease choose from {1}'.format(value, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES)))
                self.cluster_type = value

 def save(self, client=None, reload_config=True):
     """
     Saves the configuration to a given file, optionally a remote one
     :param client: If provided, save remote configuration
     :param reload_config: Reload the running Storage Driver configuration
     """
     self._validate()
     contents = json.dumps(self.configuration, indent=4)
     EtcdConfiguration.set(self.path, contents, raw=True)
     if self.config_type == 'storagedriver' and reload_config is True:
         if len(self.dirty_entries) > 0:
             if client is None:
                 logger.info('Applying local storagedriver configuration changes')
                 changes = LSRClient(self.remote_path).update_configuration(self.remote_path)
             else:
                 logger.info('Applying storagedriver configuration changes on {0}'.format(client.ip))
                 with Remote(client.ip, [LSRClient]) as remote:
                     changes = copy.deepcopy(remote.LocalStorageRouterClient(self.remote_path).update_configuration(self.remote_path))
             for change in changes:
                 if change['param_name'] not in self.dirty_entries:
                     raise RuntimeError('Unexpected configuration change: {0}'.format(change['param_name']))
                 logger.info('Changed {0} from "{1}" to "{2}"'.format(change['param_name'], change['old_value'], change['new_value']))
                 self.dirty_entries.remove(change['param_name'])
             logger.info('Changes applied')
             if len(self.dirty_entries) > 0:
                 logger.warning('Following changes were not applied: {0}'.format(', '.join(self.dirty_entries)))
         else:
             logger.debug('No need to apply changes, nothing changed')
     self.is_new = False
     self.dirty_entries = []

    def migrate(master_ips=None, extra_ips=None):
        """
        Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one
        :param master_ips: IP addresses of the MASTER nodes
        :param extra_ips: IP addresses of the EXTRA nodes
        """

        data = EtcdConfiguration.get('/ovs/framework/versions') if EtcdConfiguration.exists('/ovs/framework/versions') else {}
        migrators = []
        path = os.path.join(os.path.dirname(__file__), 'migration')
        for filename in os.listdir(path):
            if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py'):
                name = filename.replace('.py', '')
                module = imp.load_source(name, os.path.join(path, filename))
                for member in inspect.getmembers(module):
                    if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]:
                        migrators.append((member[1].identifier, member[1].migrate))

        end_version = 0
        for identifier, method in migrators:
            base_version = data[identifier] if identifier in data else 0
            version = method(base_version, master_ips, extra_ips)
            if version > end_version:
                end_version = version
            data[identifier] = end_version

        EtcdConfiguration.set('/ovs/framework/versions', data)

def teardown():
    """
    Teardown for Arakoon package, will be executed when all started tests in this package have ended
    Removal actions of possible things left over after the test-run
    :return: None
    """
    autotest_config = General.get_config()
    backend_name = autotest_config.get('backend', 'name')
    backend = GeneralBackend.get_by_name(backend_name)
    if backend is not None:
        GeneralAlba.remove_alba_backend(backend.alba_backend)

    for storagerouter in GeneralStorageRouter.get_masters():
        root_client = SSHClient(storagerouter, username='root')
        if GeneralService.get_service_status(name='ovs-scheduled-tasks',
                                             client=root_client) is False:
            GeneralService.start_service(name='ovs-scheduled-tasks',
                                         client=root_client)

        for location in TEST_CLEANUP:
            root_client.run('rm -rf {0}'.format(location))

    for key in KEY_CLEANUP:
        if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
            EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))

def run_event_consumer():
    """
    Check whether to run the event consumer
    """
    my_ip = EtcdConfiguration.get('/ovs/framework/hosts/{0}/ip'.format(System.get_my_machine_id()))
    for endpoint in EtcdConfiguration.get('/ovs/framework/messagequeue|endpoints'):
        if endpoint.startswith(my_ip):
            return True
    return False

 def delete_etcd_config(cluster_name):
     """
     Remove the etcd entry for arakoon cluster_name
     :param cluster_name: Name of the arakoon cluster
     :return: None
     """
     etcd_key = GeneralArakoon.ETCD_CONFIG_KEY.format(cluster_name)
     if EtcdConfiguration.exists(etcd_key, raw=True):
         EtcdConfiguration.delete(os.path.dirname(etcd_key))

        def _process_disk(_info, _disks, _node):
            disk = _info.get('disk')
            if disk is None:
                return
            disk_status = 'uninitialized'
            disk_status_detail = ''
            disk_alba_backend_guid = ''
            if disk['available'] is False:
                osd = _info.get('osd')
                disk_alba_state = disk['state']['state']
                if disk_alba_state == 'ok':
                    if osd is None:
                        disk_status = 'initialized'
                    elif osd['id'] is None:
                        alba_id = osd['alba_id']
                        if alba_id is None:
                            disk_status = 'available'
                        else:
                            disk_status = 'unavailable'
                            alba_backend = alba_backend_map.get(alba_id)
                            if alba_backend is not None:
                                disk_alba_backend_guid = alba_backend.guid
                    else:
                        disk_status = 'error'
                        disk_status_detail = 'communicationerror'
                        disk_alba_backend_guid = self.guid

                        for asd in _node.asds:
                            if asd.asd_id == disk['asd_id'] and asd.statistics != {}:
                                disk_status = 'warning'
                                disk_status_detail = 'recenterrors'

                                read = osd['read'] or [0]
                                write = osd['write'] or [0]
                                errors = osd['errors']
                                global_interval_key = '/ovs/alba/backends/global_gui_error_interval'
                                backend_interval_key = '/ovs/alba/backends/{0}/gui_error_interval'.format(self.guid)
                                interval = EtcdConfiguration.get(global_interval_key)
                                if EtcdConfiguration.exists(backend_interval_key):
                                    interval = EtcdConfiguration.get(backend_interval_key)
                                if len(errors) == 0 or (len(read + write) > 0 and max(min(read), min(write)) > max(error[0] for error in errors) + interval):
                                    disk_status = 'claimed'
                                    disk_status_detail = ''
                elif disk_alba_state == 'decommissioned':
                    disk_status = 'unavailable'
                    disk_status_detail = 'decommissioned'
                else:
                    disk_status = 'error'
                    disk_status_detail = disk['state']['detail']
                    alba_backend = alba_backend_map.get(osd.get('alba_id'))
                    if alba_backend is not None:
                        disk_alba_backend_guid = alba_backend.guid
            disk['status'] = disk_status
            disk['status_detail'] = disk_status_detail
            disk['alba_backend_guid'] = disk_alba_backend_guid
            _disks.append(disk)

 def get_path(binary_name):
     machine_id = System.get_my_machine_id()
     config_location = '/ovs/framework/hosts/{0}/paths|{1}'.format(machine_id, binary_name)
     path = EtcdConfiguration.get(config_location)
     if not path:
         try:
             path = check_output('which {0}'.format(binary_name), shell=True).strip()
             EtcdConfiguration.set(config_location, path)
         except CalledProcessError:
             return None
     return path

    def list(self, discover=False, ip=None, node_id=None):
        """
        Lists all available ALBA Nodes
        :param discover: If True and IP provided, return list of single ALBA node, If True and no IP provided, return all ALBA nodes else return modeled ALBA nodes
        :param ip: IP of ALBA node to retrieve
        :param node_id: ID of the ALBA node
        """
        if discover is False and (ip is not None or node_id is not None):
            raise RuntimeError('Discover is mutually exclusive with IP and nodeID')
        if (ip is None and node_id is not None) or (ip is not None and node_id is None):
            raise RuntimeError('Both IP and nodeID need to be specified')

        if discover is False:
            return AlbaNodeList.get_albanodes()

        if ip is not None:
            node = AlbaNode(volatile=True)
            node.ip = ip
            node.type = 'ASD'
            node.node_id = node_id
            node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
            node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
            node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
            data = node.client.get_metadata()
            if data['_success'] is False and data['_error'] == 'Invalid credentials':
                raise RuntimeError('Invalid credentials')
            if data['node_id'] != node_id:
                raise RuntimeError('Unexpected node identifier. {0} vs {1}'.format(data['node_id'], node_id))
            node_list = DataList(AlbaNode, {})
            node_list._executed = True
            node_list._guids = [node.guid]
            node_list._objects = {node.guid: node}
            node_list._data = {node.guid: {'guid': node.guid, 'data': node._data}}
            return node_list

        nodes = {}
        model_node_ids = [node.node_id for node in AlbaNodeList.get_albanodes()]
        found_node_ids = []
        asd_node_ids = []
        if EtcdConfiguration.dir_exists('/ovs/alba/asdnodes'):
            asd_node_ids = EtcdConfiguration.list('/ovs/alba/asdnodes')

        for node_id in asd_node_ids:
            node = AlbaNode(volatile=True)
            node.type = 'ASD'
            node.node_id = node_id
            node.ip = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|ip'.format(node_id))
            node.port = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id))
            node.username = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id))
            node.password = EtcdConfiguration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id))
            if node.node_id not in model_node_ids and node.node_id not in found_node_ids:
                nodes[node.guid] = node
                found_node_ids.append(node.node_id)
        node_list = DataList(AlbaNode, {})
        node_list._executed = True
        node_list._guids = nodes.keys()
        node_list._objects = nodes
        node_list._data = dict([(node.guid, {'guid': node.guid, 'data': node._data}) for node in nodes.values()])
        return node_list

    def write(self):
        """
        Write the metadata to Etcd
        :return: None
        """
        if self.cluster_type is None or self.cluster_type == '':
            raise ValueError('Cluster type must be defined before being able to store the cluster metadata information')

        etcd_key = ArakoonClusterMetadata.ETCD_METADATA_KEY.format(self.cluster_id)
        EtcdConfiguration.set(key=etcd_key, value={'type': self.cluster_type,
                                                   'in_use': self.in_use,
                                                   'internal': self.internal})

    def ovs_3977_maintenance_agent_test():
        """
        Test maintenance agent processes
        """
        def _get_agent_distribution(agent_name):
            result = {}
            total = 0
            for ip in alba_node_ips:
                count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
                if count:
                    count = int(count)
                else:
                    count = 0
                total += count
                result[ip] = count
            result['total'] = total

            print 'Maintenance agent distribution: {0}'.format(result)
            for ip in alba_node_ips:
                assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
                    "Agents not equally distributed!"

            return result

        backend = GeneralBackend.get_by_name(TestALBA.backend_name)
        if backend is None:
            backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
        name = backend.alba_backend.name

        alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]

        etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
        nr_of_agents = EtcdConfiguration.get(etcd_key)
        print '1. - nr of agents: {0}'.format(nr_of_agents)

        actual_nr_of_agents = _get_agent_distribution(name)['total']
        assert nr_of_agents == actual_nr_of_agents, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)

        # set nr to zero
        EtcdConfiguration.set(etcd_key, 0)
        GeneralAlba.checkup_maintenance_agents()
        assert _get_agent_distribution(name)['total'] == 0, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
        print '2. - nr of agents: {0}'.format(nr_of_agents)

        # set nr to 10
        EtcdConfiguration.set(etcd_key, 10)
        GeneralAlba.checkup_maintenance_agents()
        assert _get_agent_distribution(name)['total'] == 10, \
            'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
        print '3. - nr of agents: {0}'.format(nr_of_agents)

 def write_config(self):
     """
     Writes the configuration down to in the format expected by Arakoon
     """
     contents = RawConfigParser()
     data = self.export()
     for section in data:
         contents.add_section(section)
         for item in data[section]:
             contents.set(section, item, data[section][item])
     config_io = StringIO()
     contents.write(config_io)
     EtcdConfiguration.set(ArakoonClusterConfig.ETCD_CONFIG_KEY.format(self.cluster_id), config_io.getvalue(), raw=True)