Python swift_helper.get_swift_configs函数代码示例

def _configure_swift(client, cluster):
    swift_enable = c_helper.get_config_value(
        cluster.cluster_configs.get('general'), c_helper.ENABLE_SWIFT)
    if swift_enable:
        swift_configs = swift.get_swift_configs()
        for conf in swift_configs:
            client.params.hadoop.add(conf['name'], conf['value'])

    def test_generate_xml_configs(self, auth_url):
        auth_url.return_value = "http://localhost:5000/v2/"

        # Make a dict of swift configs to verify generated values
        swift_vals = c_helper.extract_name_values(swift.get_swift_configs())

        # Make sure that all the swift configs are in core-site
        c = c_helper.generate_xml_configs({}, ['/mnt/one'], 'localhost', None)
        doc = xml.parseString(c['core-site'])
        configuration = doc.getElementsByTagName('configuration')
        properties = xmlutils.get_property_dict(configuration[0])
        self.assertDictContainsSubset(swift_vals, properties)

        # Make sure that user values have precedence over defaults
        c = c_helper.generate_xml_configs(
            {'HDFS': {'fs.swift.service.sahara.tenant': 'fred'}},
            ['/mnt/one'], 'localhost', None)
        doc = xml.parseString(c['core-site'])
        configuration = doc.getElementsByTagName('configuration')
        properties = xmlutils.get_property_dict(configuration[0])
        mod_swift_vals = copy.copy(swift_vals)
        mod_swift_vals['fs.swift.service.sahara.tenant'] = 'fred'
        self.assertDictContainsSubset(mod_swift_vals, properties)

        # Make sure that swift confgs are left out if not enabled
        c = c_helper.generate_xml_configs(
            {'HDFS': {'fs.swift.service.sahara.tenant': 'fred'},
             'general': {'Enable Swift': False}},
            ['/mnt/one'], 'localhost', None)
        doc = xml.parseString(c['core-site'])
        configuration = doc.getElementsByTagName('configuration')
        properties = xmlutils.get_property_dict(configuration[0])
        for key in mod_swift_vals.keys():
            self.assertNotIn(key, properties)

def _get_hadoop_configs(node_group):
    cluster = node_group.cluster
    nn_hostname = vu.get_instance_hostname(vu.get_namenode(cluster))
    dirs = _get_hadoop_dirs(node_group)
    confs = {
        'Hadoop': {
            'fs.defaultFS': 'hdfs://%s:9000' % nn_hostname
        },
        'HDFS': {
            'dfs.namenode.name.dir': ','.join(dirs['hadoop_name_dirs']),
            'dfs.namenode.data.dir': ','.join(dirs['hadoop_data_dirs']),
            'dfs.hosts': '%s/dn-include' % HADOOP_CONF_DIR,
            'dfs.hosts.exclude': '%s/dn-exclude' % HADOOP_CONF_DIR
        }
    }

    res_hostname = vu.get_instance_hostname(vu.get_resourcemanager(cluster))
    if res_hostname:
        confs['YARN'] = {
            'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
            'yarn.resourcemanager.hostname': '%s' % res_hostname,
            'yarn.resourcemanager.nodes.include-path': '%s/nm-include' % (
                HADOOP_CONF_DIR),
            'yarn.resourcemanager.nodes.exclude-path': '%s/nm-exclude' % (
                HADOOP_CONF_DIR)
        }
        confs['MapReduce'] = {
            'mapreduce.framework.name': 'yarn'
        }

    oozie = vu.get_oozie(cluster)
    if oozie:
        hadoop_cfg = {
            'hadoop.proxyuser.hadoop.hosts': '*',
            'hadoop.proxyuser.hadoop.groups': 'hadoop'
        }
        confs['Hadoop'].update(hadoop_cfg)

        oozie_cfg = o_helper.get_oozie_required_xml_configs(HADOOP_CONF_DIR)
        if c_helper.is_mysql_enabled(cluster):
            oozie_cfg.update(o_helper.get_oozie_mysql_configs())

        confs['JobFlow'] = oozie_cfg

    if c_helper.get_config_value(c_helper.ENABLE_SWIFT.applicable_target,
                                 c_helper.ENABLE_SWIFT.name, cluster):
        swift_configs = {}
        for config in swift.get_swift_configs():
            swift_configs[config['name']] = config['value']

        confs['Hadoop'].update(swift_configs)

    if c_helper.is_data_locality_enabled(cluster):
        confs['Hadoop'].update(th.TOPOLOGY_CONFIG)
        confs['Hadoop'].update({"topology.script.file.name":
                                HADOOP_CONF_DIR + "/topology.sh"})

    return confs, c_helper.get_env_configs()

 def _get_core_site_props(self, context):
     result = {}
     if context.is_node_aware:
         result.update(self._get_core_site_node_aware_props())
     for conf in swift_helper.get_swift_configs():
         result[conf['name']] = conf['value']
     for conf in self._get_impersonation_props():
         result[conf['name']] = conf['value']
     return result

    def test_get_swift_configs(self):
        self.setup_context(tenant_name="test_tenant", auth_uri="http://localhost:8080/v2.0/")
        self.override_config("os_region_name", "regionOne")

        result = h.get_swift_configs()
        self.assertEqual(8, len(result))
        self.assertIn({"name": "fs.swift.service.sahara.tenant", "value": "test_tenant", "description": ""}, result)
        self.assertIn({"name": "fs.swift.service.sahara.http.port", "value": "8080", "description": ""}, result)
        self.assertIn({"name": "fs.swift.service.sahara.region", "value": "regionOne", "description": ""}, result)

 def _get_core_site_props(self, context):
     result = {
         'hadoop.proxyuser.mapr.groups': '*',
         'hadoop.proxyuser.mapr.hosts': '*',
     }
     if context.is_node_aware:
         result.update(self._get_core_site_node_aware_props())
     for conf in swift_helper.get_swift_configs():
         result[conf['name']] = conf['value']
     return result

def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
    oozie_hostname = vu.get_instance_hostname(vu.get_oozie(cluster))
    hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster))

    ng_configs = node_group.configuration()

    general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)

    all_cfg = generate_sahara_configs(cluster, node_group)

    # inserting user-defined configs
    for key, value in extract_xml_confs(ng_configs):
        all_cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = swift.get_swift_configs()
    all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT + swift_xml_confs
    mapred_all = MAPRED_DEFAULT

    if CONF.enable_data_locality:
        all_cfg.update(topology.TOPOLOGY_CONFIG)

        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()
        mapred_all += topology.vm_awareness_mapred_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(all_cfg, core_all),
        'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
        'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
    }

    if hive_hostname:
        cfg = all_cfg
        cfg_filter = HIVE_DEFAULT
        proxy_configs = cluster.cluster_configs.get('proxy_configs')
        if CONF.use_identity_api_v3 and proxy_configs:
            cfg, cfg_filter = _inject_swift_trust_info(cfg,
                                                       cfg_filter,
                                                       proxy_configs)
        xml_configs.update({'hive-site':
                            x.create_hadoop_xml(cfg, cfg_filter)})
        LOG.debug('Generated hive-site.xml for hive {host}'.format(
            host=hive_hostname))

    if oozie_hostname:
        xml_configs.update({'oozie-site':
                            x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug('Generated oozie-site.xml for oozie {host}'.format(
            host=oozie_hostname))

    return xml_configs

def _configure_swift_to_inst(instance):
    cluster = instance.node_group.cluster
    with instance.remote() as r:
        r.execute_command('sudo curl %s -o %s/hadoop-openstack.jar' % (
            c_helper.get_swift_lib_url(cluster), HADOOP_LIB_DIR))
        core_site = r.read_file_from(PATH_TO_CORE_SITE_XML)
        configs = xmlutils.parse_hadoop_xml_with_name_and_value(core_site)
        configs.extend(swift_helper.get_swift_configs())
        confs = dict((c['name'], c['value']) for c in configs)
        new_core_site = xmlutils.create_hadoop_xml(confs)
        r.write_file_to(PATH_TO_CORE_SITE_XML, new_core_site, run_as_root=True)

def get_cluster_params(cluster):
    configs = _create_ambari_configs(cluster.cluster_configs,
                                     cluster.hadoop_version)
    swift_configs = {x["name"]: x["value"]
                     for x in swift_helper.get_swift_configs()}
    configs.setdefault("core-site", {})
    configs["core-site"].update(swift_configs)
    if utils.get_instance(cluster, common.RANGER_ADMIN):
        configs.setdefault("admin-properties", {})
        configs["admin-properties"]["db_root_password"] = (
            cluster.extra["ranger_db_password"])
    return _serialize_ambari_configs(configs)

    def test_get_swift_configs(self, authUrlConfig):
        self.setup_context(tenant_name='test_tenant')
        self.override_config("os_auth_protocol", 'http')
        self.override_config("os_auth_port", '8080')
        authUrlConfig.return_value = "http://localhost:8080/v2.0/"

        result = h.get_swift_configs()
        self.assertEqual(7, len(result))
        self.assertIn({'name': "fs.swift.service.sahara.tenant",
                       'value': 'test_tenant', 'description': ''}, result)
        self.assertIn({'name': "fs.swift.service.sahara.http.port",
                       'value': '8080', 'description': ''}, result)

    def test_get_swift_configs(self, url_for_mock):
        url_for_mock.return_value = 'http://localhost:5000/v2.0'
        self.setup_context(tenant_name='test_tenant')
        self.override_config("os_region_name", 'regionOne')

        result = h.get_swift_configs()
        self.assertEqual(8, len(result))
        self.assertIn({'name': "fs.swift.service.sahara.tenant",
                       'value': 'test_tenant', 'description': ''}, result)
        self.assertIn({'name': "fs.swift.service.sahara.http.port",
                       'value': '8080', 'description': ''}, result)
        self.assertIn({'name': "fs.swift.service.sahara.region",
                       'value': 'regionOne', 'description': ''}, result)

def generate_xml_configs(cluster, node_group, hive_mysql_passwd):
    oozie_hostname = _get_hostname(utils.get_oozie(cluster))
    hive_hostname = _get_hostname(utils.get_hiveserver(cluster))

    ng_configs = node_group.configuration()

    general_cfg = get_general_configs(hive_hostname, hive_mysql_passwd)

    all_cfg = generate_sahara_configs(cluster, node_group)

    # inserting user-defined configs
    for key, value in extract_xml_confs(ng_configs):
        all_cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = swift.get_swift_configs()
    all_cfg = generate_cfg_from_general(all_cfg, ng_configs, general_cfg)

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT + swift_xml_confs
    mapred_all = MAPRED_DEFAULT

    if CONF.enable_data_locality:
        all_cfg.update(topology.TOPOLOGY_CONFIG)

        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()
        mapred_all += topology.vm_awareness_mapred_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(all_cfg, core_all),
        'mapred-site': x.create_hadoop_xml(all_cfg, mapred_all),
        'hdfs-site': x.create_hadoop_xml(all_cfg, HDFS_DEFAULT)
    }

    if hive_hostname:
        xml_configs.update({'hive-site':
                            x.create_hadoop_xml(all_cfg, HIVE_DEFAULT)})
        LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)

    if oozie_hostname:
        xml_configs.update({'oozie-site':
                            x.create_hadoop_xml(all_cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)

    return xml_configs

def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port):
    if hadoop_port is None:
        hadoop_port = 8020

    cfg = {
        'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
        'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
                                                     '/dfs/nn'),
        'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
                                                     '/dfs/dn'),
        'hadoop.tmp.dir': extract_hadoop_path(storage_path,
                                              '/dfs'),
        'dfs.hosts': '/etc/hadoop/dn.incl',
        'dfs.hosts.exclude': '/etc/hadoop/dn.excl'
    }

    # inserting user-defined configs
    for key, value in extract_hadoop_xml_confs(configs):
        cfg[key] = value

    # Add the swift defaults if they have not been set by the user
    swft_def = []
    if is_swift_enabled(configs):
        swft_def = SWIFT_DEFAULTS
        swift_configs = extract_name_values(swift.get_swift_configs())
        for key, value in six.iteritems(swift_configs):
            if key not in cfg:
                cfg[key] = value

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT + swft_def

    if CONF.enable_data_locality:
        cfg.update(topology.TOPOLOGY_CONFIG)
        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, core_all),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    return xml_configs

def get_general_configs(hive_hostname, passwd_hive_mysql):
    config = {
        ENABLE_SWIFT.name: {
            'default_value': ENABLE_SWIFT.default_value,
            'conf': extract_name_values(swift.get_swift_configs())
        },
        ENABLE_MYSQL.name: {
            'default_value': ENABLE_MYSQL.default_value,
            'conf': m_h.get_required_mysql_configs(
                hive_hostname, passwd_hive_mysql)
        }
    }
    if CONF.enable_data_locality:
        config.update({
            ENABLE_DATA_LOCALITY.name: {
                'default_value': ENABLE_DATA_LOCALITY.default_value,
                'conf': extract_name_values(topology.vm_awareness_all_config())
            }
        })
    return config

    def test_generate_xml_configs(self, auth_url):
        auth_url.return_value = "http://localhost:5000/v2/"

        # Make a dict of swift configs to verify generated values
        swift_vals = c_helper.extract_name_values(swift.get_swift_configs())

        # Make sure that all the swift configs are in core-site
        c = c_helper.generate_xml_configs({}, ["/mnt/one"], "localhost", None)
        doc = xml.parseString(c["core-site"])
        configuration = doc.getElementsByTagName("configuration")
        properties = xmlutils.get_property_dict(configuration[0])
        self.assertDictContainsSubset(swift_vals, properties)

        # Make sure that user values have precedence over defaults
        c = c_helper.generate_xml_configs(
            {"HDFS": {"fs.swift.service.sahara.tenant": "fred"}}, ["/mnt/one"], "localhost", None
        )
        doc = xml.parseString(c["core-site"])
        configuration = doc.getElementsByTagName("configuration")
        properties = xmlutils.get_property_dict(configuration[0])
        mod_swift_vals = copy.copy(swift_vals)
        mod_swift_vals["fs.swift.service.sahara.tenant"] = "fred"
        self.assertDictContainsSubset(mod_swift_vals, properties)

        # Make sure that swift confgs are left out if not enabled
        c = c_helper.generate_xml_configs(
            {"HDFS": {"fs.swift.service.sahara.tenant": "fred"}, "general": {"Enable Swift": False}},
            ["/mnt/one"],
            "localhost",
            None,
        )
        doc = xml.parseString(c["core-site"])
        configuration = doc.getElementsByTagName("configuration")
        properties = xmlutils.get_property_dict(configuration[0])
        for key in mod_swift_vals.keys():
            self.assertNotIn(key, properties)