Python parser.EasyConfigParser类代码示例

    def test_tweak_one_version(self):
        """Test tweak_one function"""
        test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')

        # test tweaking of software version (--try-software-version)
        tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()

        # checksums should be reset to empty list, only version should be changed, nothing else
        self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
        for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
            val = toy_ec_parsed[key]
            self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
            tweaked_val = tweaked_toy_ec_parsed.get(key)
            self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))

        # check behaviour if target file already exists
        error_pattern = "File exists, not overwriting it without --force"
        self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        # existing file does get overwritten when --force is used
        init_config(build_options={'force': True, 'silent': True})
        write_file(tweaked_toy_ec, '')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')

    def parse(self):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        if self.build_specs is None:
            arg_specs = {}
        elif isinstance(self.build_specs, dict):
            # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
            arg_specs = self.build_specs
        else:
            self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
        self.log.debug("Obtained specs dict %s" % arg_specs)

        self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt))
        parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt)
        parser.set_specifications(arg_specs)
        local_vars = parser.get_config_dict()
        self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)

        # make sure all mandatory parameters are defined
        # this includes both generic mandatory parameters and software-specific parameters defined via extra_options
        missing_mandatory_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_mandatory_keys:
            self.log.error("mandatory parameters not provided in %s: %s" % (self.path, missing_mandatory_keys))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        # we need toolchain to be set when we call _parse_dependency
        for key in ['toolchain'] + local_vars.keys():
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config.keys():
                if key in ['builddependencies', 'dependencies']:
                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
                elif key in ['hiddendependencies']:
                    self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
                else:
                    self[key] = local_vars[key]
                tup = (key, self[key], type(self[key]))
                self.log.info("setting config option %s: value %s (type: %s)" % tup)
            elif key in REPLACED_PARAMETERS:
                _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]

    def parse(self):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        if self.build_specs is None:
            arg_specs = {}
        elif isinstance(self.build_specs, dict):
            # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
            arg_specs = self.build_specs
        else:
            self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
        self.log.debug("Obtained specs dict %s" % arg_specs)

        parser = EasyConfigParser(self.path)
        parser.set_specifications(arg_specs)
        local_vars = parser.get_config_dict()
        self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        # we need toolchain to be set when we call _parse_dependency
        for key in ['toolchain'] + local_vars.keys():
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config.keys() + DEPRECATED_OPTIONS.keys():
                if key in ['builddependencies', 'dependencies']:
                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
                elif key in ['hiddendependencies']:
                    self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
                else:
                    self[key] = local_vars[key]
                tup = (key, self[key], type(self[key]))
                self.log.info("setting config option %s: value %s (type: %s)" % tup)

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]

    def test_v10(self):
        ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))

        self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))

        ec = ecp.get_config_dict()

        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.3')

    def test_check_value_types(self):
        """Test checking of easyconfig parameter value types."""
        test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb')
        error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*"
        ecp = EasyConfigParser(test_ec, auto_convert_value_types=False)
        self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict)

        # test default behaviour: auto-converting of mismatched value types
        ecp = EasyConfigParser(test_ec)
        ecdict = ecp.get_config_dict()
        self.assertEqual(ecdict['version'], '1.4')

    def test_v20_deps(self):
        """Test parsing of easyconfig in format v2 that includes dependencies."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'libpng')
        # first version/toolchain listed is default
        self.assertEqual(ec['version'], '1.5.10')
        self.assertEqual(ec['toolchain'], {'name': 'goolf', 'version': '1.4.10'})

        # dependencies should be parsed correctly
        deps = ec['dependencies']
        self.assertTrue(isinstance(deps[0], Dependency))
        self.assertEqual(deps[0].name(), 'zlib')
        self.assertEqual(deps[0].version(), '1.2.5')

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'goolf.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'goolf')
        self.assertEqual(ec['version'], '1.4.10')
        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})

        # dependencies should be parsed correctly
        deps = [
            # name, version, versionsuffix, toolchain
            ('GCC', '4.7.2', None, None),
            ('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}),
            ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
            ('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}),
            ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
        ]
        for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
            self.assertEqual(ec['dependencies'][i].name(), name)
            self.assertEqual(ec['dependencies'][i].version(), version)
            self.assertEqual(ec['dependencies'][i].versionsuffix(), versionsuffix)
            self.assertEqual(ec['dependencies'][i].toolchain(), toolchain)

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental

    def test_raw(self):
        """Test passing of raw contents to EasyConfigParser."""
        ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb')
        ec_txt1 = read_file(ec_file1)
        ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb')
        ec_txt2 = read_file(ec_file2)

        ecparser = EasyConfigParser(ec_file1)
        self.assertEqual(ecparser.rawcontent, ec_txt1)

        ecparser = EasyConfigParser(rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)

        # rawcontent supersedes passed filepath
        ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)
        ec = ecparser.get_config_dict()
        self.assertEqual(ec['name'], 'gzip')
        self.assertEqual(ec['toolchain']['name'], 'goolf')

        self.assertErrorRegex(EasyBuildError, "Neither filename nor rawcontent provided", EasyConfigParser)

    def parse(self, path, format_version=None):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        parser = EasyConfigParser(path, format_version=format_version)
        local_vars = parser.get_config_dict()

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" % (missing_keys, path))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        self._legacy_license(local_vars)

        for key in local_vars:
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config:
                self[key] = local_vars[key]
                self.log.info("setting config option %s: value %s" % (key, self[key]))

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]

    def test_v20(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        # this should be ok: ie the default values
        ec = ecp.get_config_dict()
        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.2')

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental

def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
    """
    Show list of supported software

    :param output_format: output format to use
    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
    :param only_installed: only retain software for which a corresponding module is available
    :return: multi-line string presenting requested info
    """
    silent = build_option('silent')

    ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or [])
    ecs = []
    cnt = len(ec_paths)
    for idx, ec_path in enumerate(ec_paths):
        # full EasyConfig instance is only required when module name is needed
        # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
        if only_installed:
            ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec']
        else:
            ec = EasyConfigParser(filename=ec_path).get_config_dict()

        ecs.append(ec)
        print_msg('\r', prefix=False, newline=False, silent=silent)
        print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
    print_msg('', prefix=False, silent=silent)

    software = {}
    for ec in ecs:
        software.setdefault(ec['name'], [])
        if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
            toolchain = DUMMY_TOOLCHAIN_NAME
        else:
            toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])

        keys = ['description', 'homepage', 'version', 'versionsuffix']

        info = {'toolchain': toolchain}
        for key in keys:
            info[key] = ec.get(key, '')

        # make sure values like homepage & versionsuffix get properly templated
        if isinstance(ec, dict):
            template_values = template_constant_dict(ec, skip_lower=False)
            for key in keys:
                if '%(' in info[key]:
                    try:
                        info[key] = info[key] % template_values
                    except (KeyError, TypeError, ValueError) as err:
                        _log.debug("Ignoring failure to resolve templates: %s", err)

        software[ec['name']].append(info)

        if only_installed:
            software[ec['name']][-1].update({'mod_name': ec.full_mod_name})

    print_msg("Found %d different software packages" % len(software), silent=silent)

    if only_installed:
        avail_mod_names = modules_tool().available()

        # rebuild software, only retain entries with a corresponding available module
        software, all_software = {}, software
        for key in all_software:
            for entry in all_software[key]:
                if entry['mod_name'] in avail_mod_names:
                    software.setdefault(key, []).append(entry)

        print_msg("Retained %d installed software packages" % len(software), silent=silent)

    return generate_doc('list_software_%s' % output_format, [software, detailed])