Python utilities.init_config函数代码示例

    def test_package(self):
        """Test package function."""
        init_config(build_options={'silent': True})

        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
        ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False)

        mock_fpm(self.test_prefix)

        # import needs to be done here, since test easyblocks are only included later
        from easybuild.easyblocks.toy import EB_toy
        easyblock = EB_toy(ec)

        # build & install first
        easyblock.run_all_steps(False)

        # package using default packaging configuration (FPM to build RPM packages)
        pkgdir = package(easyblock)

        pkgfile = os.path.join(pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION)
        self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile)

        pkgtxt = read_file(pkgfile)
        pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir)
        self.assertTrue(pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))

        if DEBUG:
            print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE))

    def test_log_file_format(self):
        """Test for log_file_format()."""

        # first test defaults -> no templating when no values are provided
        self.assertEqual(log_file_format(), 'easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log')
        self.assertEqual(log_file_format(return_directory=True), 'easybuild')

        # test whether provided values are used to complete template
        ec = {'name': 'foo', 'version': '1.2.3'}
        res = log_file_format(ec=ec, date='20190322', timestamp='094356')
        self.assertEqual(res, 'easybuild-foo-1.2.3-20190322.094356.log')

        res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356')
        self.assertEqual(res, 'easybuild')

        # partial templating is done when only some values are provided...
        self.assertEqual(log_file_format(ec=ec), 'easybuild-foo-1.2.3-%(date)s.%(time)s.log')
        res = log_file_format(date='20190322', timestamp='094356')
        self.assertEqual(res, 'easybuild-%(name)s-%(version)s-20190322.094356.log')

        # also try with a custom setting
        init_config(args=['--logfile-format=eb-%(name)s-%(date)s,log-%(version)s-%(date)s-%(time)s.out'])
        self.assertEqual(log_file_format(), 'log-%(version)s-%(date)s-%(time)s.out')
        self.assertEqual(log_file_format(return_directory=True), 'eb-%(name)s-%(date)s')

        res = log_file_format(ec=ec, date='20190322', timestamp='094356')
        self.assertEqual(res, 'log-1.2.3-20190322-094356.out')

        res = log_file_format(return_directory=True, ec=ec, date='20190322', timestamp='094356')
        self.assertEqual(res, 'eb-foo-20190322')

        # test handling of incorrect setting for --logfile-format
        init_config(args=['--logfile-format=easybuild,log.txt,thisiswrong'])
        error_pattern = "Incorrect log file format specification, should be 2-tuple"
        self.assertErrorRegex(EasyBuildError, error_pattern, log_file_format)

    def test_close_pr(self):
        """Test close_pr function."""
        if self.skip_github_tests:
            print "Skipping test_close_pr, no GitHub token available?"
            return

        build_options = {
            'dry_run': True,
            'github_user': GITHUB_TEST_ACCOUNT,
            'pr_target_account': GITHUB_USER,
            'pr_target_repo': GITHUB_REPO,
        }
        init_config(build_options=build_options)

        self.mock_stdout(True)
        gh.close_pr(2, motivation_msg='just a test')
        stdout = self.get_stdout()
        self.mock_stdout(False)

        patterns = [
            "hpcugent/testrepository PR #2 was submitted by migueldiascosta",
            "[DRY RUN] Adding comment to testrepository issue #2: '" +
            "@migueldiascosta, this PR is being closed for the following reason(s): just a test",
            "[DRY RUN] Closed hpcugent/testrepository pull request #2",
        ]
        for pattern in patterns:
            self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout))

    def test_check_capability_mapping(self):
        """Test comparing the functionality of two toolchains"""
        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        init_config(build_options={
            'valid_module_classes': module_classes(),
            'robot_path': test_easyconfigs,
        })
        get_toolchain_hierarchy.clear()
        foss_hierarchy = get_toolchain_hierarchy({'name': 'foss', 'version': '2018a'}, incl_capabilities=True)
        iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '2016.01'},
                                                  incl_capabilities=True)

        # Hierarchies are returned with top-level toolchain last, foss has 4 elements here, intel has 2
        self.assertEqual(foss_hierarchy[0]['name'], 'GCC')
        self.assertEqual(foss_hierarchy[1]['name'], 'golf')
        self.assertEqual(foss_hierarchy[2]['name'], 'gompi')
        self.assertEqual(foss_hierarchy[3]['name'], 'foss')
        self.assertEqual(iimpi_hierarchy[0]['name'], 'GCCcore')
        self.assertEqual(iimpi_hierarchy[1]['name'], 'iccifort')
        self.assertEqual(iimpi_hierarchy[2]['name'], 'iimpi')

        # golf <-> iimpi (should return False)
        self.assertFalse(check_capability_mapping(foss_hierarchy[1], iimpi_hierarchy[1]), "golf requires math libs")
        # gompi <-> iimpi
        self.assertTrue(check_capability_mapping(foss_hierarchy[2], iimpi_hierarchy[2]))
        # GCC <-> iimpi
        self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[2]))
        # GCC <-> iccifort
        self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[1]))
        # GCC <-> GCCcore
        self.assertTrue(check_capability_mapping(foss_hierarchy[0], iimpi_hierarchy[0]))

    def test_read_write_file(self):
        """Test reading/writing files."""

        fp = os.path.join(self.test_prefix, 'test.txt')
        txt = "test123"
        ft.write_file(fp, txt)
        self.assertEqual(ft.read_file(fp), txt)

        txt2 = '\n'.join(['test', '123'])
        ft.write_file(fp, txt2, append=True)
        self.assertEqual(ft.read_file(fp), txt+txt2)

        # also test behaviour of write_file under --dry-run
        build_options = {
            'extended_dry_run': True,
            'silent': False,
        }
        init_config(build_options=build_options)

        foo = os.path.join(self.test_prefix, 'foo.txt')

        self.mock_stdout(True)
        ft.write_file(foo, 'bar')
        txt = self.get_stdout()
        self.mock_stdout(False)

        self.assertFalse(os.path.exists(foo))
        self.assertTrue(re.match("^file written: .*/foo.txt$", txt))

        ft.write_file(foo, 'bar', forced=True)
        self.assertTrue(os.path.exists(foo))
        self.assertEqual(ft.read_file(foo), 'bar')

    def test_check_readiness(self):
        """Test check_readiness method."""
        init_config(build_options={'validate': False})

        # check that check_readiness step works (adding dependencies, etc.)
        ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb'
        topdir = os.path.dirname(os.path.abspath(__file__))
        ec_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 'o', 'OpenMPI', ec_file)
        ec = EasyConfig(ec_path)
        eb = EasyBlock(ec)
        eb.check_readiness_step()

        # a proper error should be thrown for dependencies that can't be resolved (module should be there)
        tmpdir = tempfile.mkdtemp()
        shutil.copy2(ec_path, tmpdir)
        ec_path = os.path.join(tmpdir, ec_file)
        f = open(ec_path, 'a')
        f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n")
        f.close()
        ec = EasyConfig(ec_path)
        eb = EasyBlock(ec)
        try:
            eb.check_readiness_step()
        except EasyBuildError, err:
            err_regex = re.compile("Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4")
            self.assertTrue(err_regex.search(str(err)), "Pattern '%s' found in '%s'" % (err_regex.pattern, err))

    def test_robot_archived_easyconfigs(self):
        """Test whether robot can pick up archived easyconfigs when asked."""
        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')

        gzip_ec = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.5-ictce-4.1.13.eb')
        gzip_ectxt = read_file(gzip_ec)

        test_ec = os.path.join(self.test_prefix, 'test.eb')
        tc_spec = "toolchain = {'name': 'ictce', 'version': '3.2.2.u3'}"
        regex = re.compile("^toolchain = .*", re.M)
        test_ectxt = regex.sub(tc_spec, gzip_ectxt)
        write_file(test_ec, test_ectxt)
        ecs, _ = parse_easyconfigs([(test_ec, False)])
        self.assertErrorRegex(EasyBuildError, "Irresolvable dependencies encountered", resolve_dependencies,
                              ecs, self.modtool, retain_all_deps=True)

        # --consider-archived-easyconfigs must be used to let robot pick up archived easyconfigs
        init_config(build_options={
            'consider_archived_easyconfigs': True,
            'robot_path': [test_ecs],
        })
        res = resolve_dependencies(ecs, self.modtool, retain_all_deps=True)
        self.assertEqual([ec['full_mod_name'] for ec in res], ['ictce/3.2.2.u3', 'gzip/1.5-ictce-3.2.2.u3'])
        expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb')
        self.assertTrue(os.path.samefile(res[0]['spec'], expected))

    def test_module_mismatch(self):
        """Test whether mismatch detection between modules tool and 'module' function works."""
        # redefine 'module' function (deliberate mismatch with used module command in MockModulesTool)
        os.environ["module"] = "() {  eval `/tmp/Modules/$MODULE_VERSION/bin/modulecmd bash $*`\n}"
        error_regex = ".*pattern .* not found in defined 'module' function"
        self.assertErrorRegex(EasyBuildError, error_regex, MockModulesTool, testing=True)

        # check whether escaping error by allowing mismatch via build options works
        build_options = {"allow_modules_tool_mismatch": True}
        init_config(build_options=build_options)

        fancylogger.logToFile(self.logfile)

        mt = MockModulesTool(testing=True)
        f = open(self.logfile, "r")
        logtxt = f.read()
        f.close()
        warn_regex = re.compile("WARNING .*pattern .* not found in defined 'module' function")
        self.assertTrue(warn_regex.search(logtxt), "Found pattern '%s' in: %s" % (warn_regex.pattern, logtxt))

        # redefine 'module' function with correct module command
        os.environ["module"] = "() {  eval `/bin/echo $*`\n}"
        mt = MockModulesTool(testing=True)
        self.assertTrue(isinstance(mt.loaded_modules(), list))  # dummy usage

        # a warning should be logged if the 'module' function is undefined
        del os.environ["module"]
        mt = MockModulesTool(testing=True)
        f = open(self.logfile, "r")
        logtxt = f.read()
        f.close()
        warn_regex = re.compile("WARNING No 'module' function defined, can't check if it matches .*")
        self.assertTrue(warn_regex.search(logtxt), "Pattern %s found in %s" % (warn_regex.pattern, logtxt))

        fancylogger.logToFile(self.logfile, enable=False)

    def test_lmod_specific(self):
        """Lmod-specific test (skipped unless Lmod is used as modules tool)."""
        lmod_abspath = which(Lmod.COMMAND)
        # only run this test if 'lmod' is available in $PATH
        if lmod_abspath is not None:
            build_options = {
                'allow_modules_tool_mismatch': True,
            }
            init_config(build_options=build_options)

            # drop any location where 'lmod' or 'spider' can be found from $PATH
            paths = os.environ.get('PATH', '').split(os.pathsep)
            new_paths = []
            for path in paths:
                lmod_cand_path = os.path.join(path, Lmod.COMMAND)
                spider_cand_path = os.path.join(path, 'spider')
                if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path):
                    new_paths.append(path)
            os.environ['PATH'] = os.pathsep.join(new_paths)

            # make sure $MODULEPATH contains path that provides some modules
            os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))

            # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD
            os.environ['LMOD_CMD'] = lmod_abspath
            lmod = Lmod(testing=True)

            # obtain list of availabe modules, should be non-empty
            self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty")

            # test updating local spider cache (but don't actually update the local cache file!)
            self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")

    def test_external_modules_metadata(self):
        """Test --external-modules-metadata."""
        # empty list by default
        cfg = init_config()
        self.assertEqual(cfg.external_modules_metadata, [])

        testcfgtxt = EXTERNAL_MODULES_METADATA
        testcfg = os.path.join(self.test_prefix, 'test_external_modules_metadata.cfg')
        write_file(testcfg, testcfgtxt)

        cfg = init_config(args=['--external-modules-metadata=%s' % testcfg])

        netcdf = {
            'name': ['netCDF', 'netCDF-Fortran'],
            'version': ['4.3.2', '4.3.2'],
            'prefix': 'NETCDF_DIR',
        }
        self.assertEqual(cfg.external_modules_metadata['cray-netcdf/4.3.2'], netcdf)
        hdf5 = {
            'name': ['HDF5'],
            'version': ['1.8.13'],
            'prefix': 'HDF5_DIR',
        }
        self.assertEqual(cfg.external_modules_metadata['cray-hdf5/1.8.13'], hdf5)

        # impartial metadata is fine
        self.assertEqual(cfg.external_modules_metadata['foo'], {'name': ['Foo'], 'prefix': '/foo'})
        self.assertEqual(cfg.external_modules_metadata['bar/1.2.3'], {'name': ['bar'], 'version': ['1.2.3']})

        # if both names and versions are specified, lists must have same lengths
        write_file(testcfg, '\n'.join(['[foo/1.2.3]', 'name = foo,bar', 'version = 1.2.3']))
        args = ['--external-modules-metadata=%s' % testcfg]
        err_msg = "Different length for lists of names/versions in metadata for external module"
        self.assertErrorRegex(EasyBuildError, err_msg, init_config, args=args)

    def test_toolchain_prepare_rpath(self):
        """Test toolchain.prepare under --rpath"""

        # put fake 'gcc' command in place that just echos its arguments
        fake_gcc = os.path.join(self.test_prefix, 'fake', 'gcc')
        write_file(fake_gcc, '#!/bin/bash\necho "[email protected]"')
        adjust_permissions(fake_gcc, stat.S_IXUSR)
        os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', ''))

        # enable --rpath and prepare toolchain
        init_config(build_options={'rpath': True, 'rpath_filter': ['/ba.*']})
        tc = self.get_toolchain('gompi', version='1.3.12')

        # preparing RPATH wrappers requires --experimental, need to bypass that here
        tc.log.experimental = lambda x: x

        tc.prepare()

        # check whether fake gcc was wrapped and that arguments are what they should be
        # no -rpath for /bar because of rpath filter
        out, _ = run_cmd('gcc ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
        expected = ' '.join([
            '-Wl,-rpath=$ORIGIN/../lib',
            '-Wl,-rpath=$ORIGIN/../lib64',
            '-Wl,--disable-new-dtags',
            '-Wl,-rpath=/foo',
            '%(user)s.c',
            '-L/foo',
            '-L/bar',
            '$FOO',
            '-DX=""',
        ])
        self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})

    def test_setvar(self):
        """Test setvar function."""
        self.mock_stdout(True)
        env.setvar('FOO', 'bar')
        txt = self.get_stdout()
        self.mock_stdout(False)
        self.assertEqual(os.getenv('FOO'), 'bar')
        self.assertEqual(os.environ['FOO'], 'bar')
        # no printing if dry run is not enabled
        self.assertEqual(txt, '')

        build_options = {
            'extended_dry_run': True,
            'silent': False,
        }
        init_config(build_options=build_options)
        self.mock_stdout(True)
        env.setvar('FOO', 'foobaz')
        txt = self.get_stdout()
        self.mock_stdout(False)
        self.assertEqual(os.getenv('FOO'), 'foobaz')
        self.assertEqual(os.environ['FOO'], 'foobaz')
        self.assertEqual(txt, "  export FOO=\"foobaz\"\n")

        # disabling verbose
        self.mock_stdout(True)
        env.setvar('FOO', 'barfoo', verbose=False)
        txt = self.get_stdout()
        self.mock_stdout(False)
        self.assertEqual(os.getenv('FOO'), 'barfoo')
        self.assertEqual(os.environ['FOO'], 'barfoo')
        self.assertEqual(txt, '')

    def test_map_easyconfig_to_target_tc_hierarchy(self):
        """Test mapping of easyconfig to target hierarchy"""
        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        init_config(build_options={
            'robot_path': test_easyconfigs,
            'silent': True,
            'valid_module_classes': module_classes(),
        })
        get_toolchain_hierarchy.clear()

        gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'}
        iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
        # The below mapping includes a binutils mapping (2.26 to 2.25)
        tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool)
        ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb')
        tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping)
        tweaked_ec = process_easyconfig(tweaked_spec)[0]
        tweaked_dict = tweaked_ec['ec'].asdict()
        # First check the mapped toolchain
        key, value = 'toolchain', iccifort_binutils_tc
        self.assertTrue(key in tweaked_dict and value == tweaked_dict[key])
        # Also check that binutils has been mapped
        for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items():
            self.assertTrue(key in tweaked_dict['builddependencies'][0] and
                            value == tweaked_dict['builddependencies'][0][key])

    def test_get_toolchain_hierarchy(self):
        """Test get_toolchain_hierarchy function."""
        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
        init_config(build_options={
            'valid_module_classes': module_classes(),
            'robot_path': test_easyconfigs,
        })

        goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
        self.assertEqual(goolf_hierarchy, [
            {'name': 'GCC', 'version': '4.7.2'},
            {'name': 'gompi', 'version': '1.4.10'},
            {'name': 'goolf', 'version': '1.4.10'},
        ])

        iimpi_hierarchy = get_toolchain_hierarchy({'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'})
        self.assertEqual(iimpi_hierarchy, [
            {'name': 'iccifort', 'version': '2013.5.192-GCC-4.8.3'},
            {'name': 'iimpi', 'version': '5.5.3-GCC-4.8.3'},
        ])

        # test also including dummy
        init_config(build_options={
            'add_dummy_to_minimal_toolchains': True,
            'valid_module_classes': module_classes(),
            'robot_path': test_easyconfigs,
        })

        # testing with gompi/1.4.10, since the result for goolf/1.4.10 is cached (and it's hard to reset the cache)
        gompi_hierarchy = get_toolchain_hierarchy({'name': 'gompi', 'version': '1.4.10'})
        self.assertEqual(gompi_hierarchy, [
            {'name': 'dummy', 'version': ''},
            {'name': 'GCC', 'version': '4.7.2'},
            {'name': 'gompi', 'version': '1.4.10'},
        ])

    def test_remove_file(self):
        """Test remove_file"""
        testfile = os.path.join(self.test_prefix, 'foo')
        ft.write_file(testfile, 'bar')

        self.assertTrue(os.path.exists(testfile))
        ft.remove_file(testfile)

        ft.write_file(testfile, 'bar')
        ft.adjust_permissions(self.test_prefix, stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH, add=False)
        self.assertErrorRegex(EasyBuildError, "Failed to remove", ft.remove_file, testfile)

        # also test behaviour of remove_file under --dry-run
        build_options = {
            'extended_dry_run': True,
            'silent': False,
        }
        init_config(build_options=build_options)
        self.mock_stdout(True)
        ft.remove_file(testfile)
        txt = self.get_stdout()
        self.mock_stdout(False)

        regex = re.compile("^file [^ ]* removed$")
        self.assertTrue(regex.match(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))