Python Machines.get_default_MPIlib方法代码示例

# 需要导入模块: from CIME.XML.machines import Machines [as 别名]
# 或者: from CIME.XML.machines.Machines import get_default_MPIlib [as 别名]
def _main():
    output, build_dir, build_optimized, clean,\
        cmake_args, compiler, enable_genf90, machine, machines_dir,\
        make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
        use_openmp, xml_test_list, verbose \
        = parse_command_line(sys.argv)

#=================================================
# Find directory and file paths.
#=================================================
    suite_specs = []
    # TODO: this violates cime policy of direct access to xml
    # should be moved to CIME/XML
    if xml_test_list is not None:
        test_xml_tree = ElementTree()
        test_xml_tree.parse(xml_test_list)
        known_paths = {
            "here": os.path.abspath(os.path.dirname(xml_test_list)),
            }
        suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
    if test_spec_dir is not None:
        suite_specs.append(
            TestSuiteSpec("__command_line_test__",
                          ["__command_line_test__"],
                          [os.path.abspath(test_spec_dir)])
            )


    if machines_dir is not None:
        machines_file = os.path.join(machines_dir, "config_machines.xml")
        machobj = Machines(infile=machines_file, machine=machine)
    else:
        machobj = Machines(machine=machine)

    # Create build directory if necessary.
    build_dir = os.path.abspath(build_dir)

    if not os.path.isdir(build_dir):
        os.mkdir(build_dir)

    # Switch to the build directory.
    os.chdir(build_dir)

    #=================================================
    # Functions to perform various stages of build.
    #=================================================

    if not use_mpi:
        mpilib = "mpi-serial"
    elif mpilib is None:
        mpilib = machobj.get_default_MPIlib()
        logger.info("Using mpilib: {}".format(mpilib))

    if compiler is None:
        compiler = machobj.get_default_compiler()
        logger.info("Compiler is {}".format(compiler))

    compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)

    pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp)

    debug = not build_optimized
    os_ = machobj.get_value("OS")

    # Create the environment, and the Macros.cmake file
    #
    #
    configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_,
              unit_testing=True)
    machspecific = EnvMachSpecific(build_dir, unit_testing=True)

    fake_case = FakeCase(compiler, mpilib, debug)
    machspecific.load_env(fake_case)
    os.environ["OS"] = os_
    os.environ["COMPILER"] = compiler
    os.environ["DEBUG"] = stringify_bool(debug)
    os.environ["MPILIB"] = mpilib
    if use_openmp:
        os.environ["compile_threaded"] = "true"
    else:
        os.environ["compile_threaded"] = "false"

    os.environ["UNIT_TEST_HOST"] = socket.gethostname()
    if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
        # The CMake Netcdf find utility that we use (from pio2) seems to key off
        # of the environment variable NETCDF, but not NETCDF_PATH
        logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"]))
        os.environ["NETCDF"] = os.environ["NETCDF_PATH"]

    if not use_mpi:
        mpirun_command = ""
    elif mpirun_command is None:
        mpi_attribs = {
            "compiler" : compiler,
            "mpilib"   : mpilib,
            "threaded" : use_openmp,
            "unit_testing" : True
        }

        # We can get away with specifying case=None since we're using exe_only=True
#.........这里部分代码省略.........

# 需要导入模块: from CIME.XML.machines import Machines [as 别名]
# 或者: from CIME.XML.machines.Machines import get_default_MPIlib [as 别名]
def load_balancing_submit(compset, res, pesfile, mpilib, compiler, project, machine,
                          extra_options_file, test_id, force_purge, test_root):
################################################################################
    # Read in list of pes from given file
    expect(os.access(pesfile, os.R_OK), 'ERROR: File %s not found', pesfile)

    logger.info('Reading XML file %s. Searching for pesize entries:', pesfile)
    try:
        pesobj = Pes(pesfile)
    except ParseError:
        expect(False, 'ERROR: File %s not parseable', pesfile)

    pesize_list = []
    grid_nodes = pesobj.get_children("grid")
    for gnode in grid_nodes:
        mach_nodes = pesobj.get_children("mach", root=gnode)
        for mnode in mach_nodes:
            pes_nodes = pesobj.get_children("pes", root=mnode)
            for pnode in pes_nodes:
                pesize = pesobj.get(pnode, 'pesize')
                if not pesize:
                    logger.critical('No pesize for pes node in file %s', pesfile)
                if pesize in pesize_list:
                    logger.critical('pesize %s duplicated in file %s', pesize, pesfile)
                pesize_list.append(pesize)

    expect(pesize_list, 'ERROR: No grid entries found in pes file {}'.format(pesfile))

    machobj = Machines(machine=machine)
    if test_root is None:
        test_root = machobj.get_value("CIME_OUTPUT_ROOT")
    if machine is None:
        machine = machobj.get_machine_name()
        print "machine is {}".format(machine)
    if compiler is None:
        compiler = machobj.get_default_compiler()
        print "compiler is {}".format(compiler)
    if mpilib is None:
        mpilib = machobj.get_default_MPIlib({"compiler":compiler})




    test_names = []
    for i in xrange(len(pesize_list)):
        test_names.append(get_full_test_name("PFS_I{}".format(i),grid=res, compset=compset,
                                             machine=machine, compiler=compiler))
        casedir = os.path.join(test_root, test_names[-1] + "." + test_id)
        print "casedir is {}".format(casedir)
        if os.path.isdir(casedir):
            if force_purge:
                logger.info('Removing directory %s', casedir)
                shutil.rmtree(casedir)
            else:
                expect(False,
                       "casedir {} already exists, use the --force-purge option, --test-root or"
                       " --test-id options".format(casedir))

    tests = TestScheduler(test_names, no_setup = True,
                          compiler=compiler, machine_name=machine, mpilib=mpilib,
                          test_root=test_root, test_id=test_id, project=project)
    success = tests.run_tests(wait=True)
    expect(success, "Error in creating cases")
    testnames = []
    for test in tests.get_testnames():
        testname =  os.path.join(test_root, test + "." + test_id)
        testnames.append( testname)
        logger.info("test is {}".format(testname))
        with Case(testname) as case:
            pes_ntasks, pes_nthrds, pes_rootpe, _, _, _ = \
                                                    pesobj.find_pes_layout('any', 'any', 'any', pesize_opts=pesize_list.pop(0))
            for key in pes_ntasks:
                case.set_value(key, pes_ntasks[key])
            for key in pes_nthrds:
                case.set_value(key, pes_nthrds[key])
            for key in pes_rootpe:
                case.set_value(key, pes_rootpe[key])

            if extra_options_file is not None:
                try:
                    extras = open(extra_options_file, 'r')
                    for line in extras.readlines():
                        split = line.split('=')
                        if len(split) == 2:
                            logger.info('setting %s=%s', split[0], split[1])
                            case.set_value(split[0], split[1])
                        else:
                            logger.debug('ignoring line in {}: {}'.format(
                                extra_options_file, line))
                    extras.close()
                except IOError:
                    expect(False, "ERROR: Could not read file {}".format(extra_options_file))


    tests = TestScheduler(test_names, use_existing=True, test_root=test_root, test_id=test_id)
    success = tests.run_tests(wait=False)
    expect(success, "Error in running cases")

    # need to fix
    logger.info('Timing jobs submitted. After jobs completed, run to optimize '
#.........这里部分代码省略.........

# 需要导入模块: from CIME.XML.machines import Machines [as 别名]
# 或者: from CIME.XML.machines.Machines import get_default_MPIlib [as 别名]
    def configure(self, compset_name, grid_name, machine_name=None,
                  project=None, pecount=None, compiler=None, mpilib=None,
                  user_compset=False, pesfile=None,
                  user_grid=False, gridfile=None, ninst=1, test=False,
                  walltime=None, queue=None):

        #--------------------------------------------
        # compset, pesfile, and compset components
        #--------------------------------------------
        self._set_compset_and_pesfile(compset_name, user_compset=user_compset, pesfile=pesfile)

        self._components = self.get_compset_components()
        #FIXME - if --user-compset is True then need to determine that
        #all of the compset settings are valid

        #--------------------------------------------
        # grid
        #--------------------------------------------
        if user_grid is True and gridfile is not None:
            self.set_value("GRIDS_SPEC_FILE", gridfile)
        grids = Grids(gridfile)

        gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname)

        self._gridname = gridinfo["GRID"]
        for key,value in gridinfo.items():
            logger.debug("Set grid %s %s"%(key,value))
            self.set_lookup_value(key,value)

        #--------------------------------------------
        # component config data
        #--------------------------------------------
        self._get_component_config_data()

        self.get_compset_var_settings()

        #--------------------------------------------
        # machine
        #--------------------------------------------
        # set machine values in env_xxx files
        machobj = Machines(machine=machine_name)
        machine_name = machobj.get_machine_name()
        self.set_value("MACH",machine_name)
        nodenames = machobj.get_node_names()
        nodenames =  [x for x in nodenames if
                      '_system' not in x and '_variables' not in x and 'mpirun' not in x and\
                      'COMPILER' not in x and 'MPILIB' not in x]

        for nodename in nodenames:
            value = machobj.get_value(nodename, resolved=False)
            type_str = self.get_type_info(nodename)
            if type_str is not None:
                logger.debug("machine nodname %s value %s"%(nodename, value))
                self.set_value(nodename, convert_to_type(value, type_str, nodename))

        if compiler is None:
            compiler = machobj.get_default_compiler()
        else:
            expect(machobj.is_valid_compiler(compiler),
                   "compiler %s is not supported on machine %s" %(compiler, machine_name))

        self.set_value("COMPILER",compiler)

        if mpilib is None:
            mpilib = machobj.get_default_MPIlib({"compiler":compiler})
        else:
            expect(machobj.is_valid_MPIlib(mpilib, {"compiler":compiler}),
                   "MPIlib %s is not supported on machine %s" %(mpilib, machine_name))
        self.set_value("MPILIB",mpilib)

        machdir = machobj.get_machines_dir()
        self.set_value("MACHDIR", machdir)

        # Create env_mach_specific settings from machine info.
        env_mach_specific_obj = self.get_env("mach_specific")
        env_mach_specific_obj.populate(machobj)
        self.schedule_rewrite(env_mach_specific_obj)

        #--------------------------------------------
        # pe payout
        #--------------------------------------------
        match1 = re.match('([0-9]+)x([0-9]+)', "" if pecount is None else pecount)
        match2 = re.match('([0-9]+)', "" if pecount is None else pecount)
        pes_ntasks = {}
        pes_nthrds = {}
        pes_rootpe = {}
        if match1:
            opti_tasks = match1.group(1)
            opti_thrds = match1.group(2)
        elif match2:
            opti_tasks = match2.group(1)
            opti_thrds = 1

        other = {}
        if match1 or match2:
            for component_class in self._component_classes:
                if component_class == "DRV":
                    component_class = "CPL"
                string = "NTASKS_" + component_class
                pes_ntasks[string] = opti_tasks
#.........这里部分代码省略.........

# 需要导入模块: from CIME.XML.machines import Machines [as 别名]
# 或者: from CIME.XML.machines.Machines import get_default_MPIlib [as 别名]
    def configure(self, compset_name, grid_name, machine_name=None,
                  project=None, pecount=None, compiler=None, mpilib=None,
                  user_compset=False, pesfile=None,
                  user_grid=False, gridfile=None, ninst=1, test=False):

        #--------------------------------------------
        # compset, pesfile, and compset components
        #--------------------------------------------
        self._set_compset_and_pesfile(compset_name, user_compset=user_compset, pesfile=pesfile)

        self._components = self.get_compset_components()
        #FIXME - if --user-compset is True then need to determine that
        #all of the compset settings are valid

        #--------------------------------------------
        # grid
        #--------------------------------------------
        if user_grid is True and gridfile is not None:
            self.set_value("GRIDS_SPEC_FILE", gridfile);
        grids = Grids(gridfile)

        gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname)
        self._gridname = gridinfo["GRID"]
        for key,value in gridinfo.items():
            logger.debug("Set grid %s %s"%(key,value))
            self.set_value(key,value)

        #--------------------------------------------
        # component config data
        #--------------------------------------------
        self._get_component_config_data()

        self.get_compset_var_settings()

        # Add the group and elements for the config_files.xml
        for idx, config_file in enumerate(self._component_config_files):
            self.set_value(config_file[0],config_file[1])

        #--------------------------------------------
        # machine
        #--------------------------------------------
        # set machine values in env_xxx files
        machobj = Machines(machine=machine_name)
        machine_name = machobj.get_machine_name()
        self.set_value("MACH",machine_name)
        nodenames = machobj.get_node_names()
        nodenames =  [x for x in nodenames if
                      '_system' not in x and '_variables' not in x and 'mpirun' not in x and\
                      'COMPILER' not in x and 'MPILIB' not in x]

        for nodename in nodenames:
            value = machobj.get_value(nodename)
            type_str = self.get_type_info(nodename)
            if type_str is not None:
                self.set_value(nodename, convert_to_type(value, type_str, nodename))

        if compiler is None:
            compiler = machobj.get_default_compiler()
        else:
            expect(machobj.is_valid_compiler(compiler),
                   "compiler %s is not supported on machine %s" %(compiler, machine_name))

        self.set_value("COMPILER",compiler)

        if mpilib is None:
            mpilib = machobj.get_default_MPIlib({"compiler":compiler})
        else:
            expect(machobj.is_valid_MPIlib(mpilib, {"compiler":compiler}),
                   "MPIlib %s is not supported on machine %s" %(mpilib, machine_name))
        self.set_value("MPILIB",mpilib)

        machdir = machobj.get_machines_dir()
        self.set_value("MACHDIR", machdir)

        # Overwriting an existing exeroot or rundir can cause problems
        exeroot = self.get_value("EXEROOT")
        rundir = self.get_value("RUNDIR")
        for wdir in (exeroot, rundir):
            if os.path.exists(wdir):
                expect(not test, "Directory %s already exists, aborting test"% wdir)
                response = raw_input("\nDirectory %s already exists, (r)eplace, (a)bort, or (u)se existing?"% wdir)
                if response.startswith("r"):
                    shutil.rmtree(wdir)
                else:
                    expect(response.startswith("u"), "Aborting by user request")

        # the following go into the env_mach_specific file
        vars = ("module_system", "environment_variables", "mpirun")
        env_mach_specific_obj = self._get_env("mach_specific")
        for var in vars:
            nodes = machobj.get_first_child_nodes(var)
            for node in nodes:
                env_mach_specific_obj.add_child(node)

        #--------------------------------------------
        # pe payout
        #--------------------------------------------
        pesobj = Pes(self._pesfile)

        #FIXME - add pesize_opts as optional argument below
#.........这里部分代码省略.........