Python machines.Machines类代码示例

    def _case_two_setup(self):
        mach_name = self._case.get_value("MACH")
        mach_obj = Machines(machine=mach_name)
        if mach_obj.is_valid_MPIlib("mpi-serial"):
            self._case.set_value("MPILIB","mpi-serial")
        else:
            logger.warning("mpi-serial is not supported on machine '{}', "
                           "so we have to fall back to default MPI and "
                           "therefore very little is being tested".format(mach_name))

        if os.path.isfile("Macros"):
            os.remove("Macros")
        self._case.case_setup(test_mode=True, reset=True)

def get_test_suite(suite, machine=None, compiler=None):
###############################################################################
    """
    Return a list of FULL test names for a suite.
    """
    expect(suite in _ALL_TESTS, "Unknown test suite: '{}'".format(suite))
    machobj = Machines(machine=machine)
    machine = machobj.get_machine_name()

    if(compiler is None):
        compiler = machobj.get_default_compiler()
    expect(machobj.is_valid_compiler(compiler),"Compiler {} not valid for machine {}".format(compiler,machine))

    inherits_from, _, tests_raw = _ALL_TESTS[suite]
    tests = []
    for item in tests_raw:
        test_mod = None
        if (isinstance(item, six.string_types)):
            test_name = item
        else:
            expect(isinstance(item, tuple), "Bad item type for item '{}'".format(str(item)))
            expect(len(item) in [2, 3], "Expected two or three items in item '{}'".format(str(item)))
            expect(isinstance(item[0], six.string_types), "Expected string in first field of item '{}'".format(str(item)))
            expect(isinstance(item[1], six.string_types), "Expected string in second field of item '{}'".format(str(item)))

            test_name = item[0]
            if (len(item) == 2):
                test_mod = item[1]
            else:
                expect(type(item[2]) in [six.string_types, tuple], "Expected string or tuple for third field of item '{}'".format(str(item)))
                test_mod_machines = [item[2]] if isinstance(item[2], six.string_types) else item[2]
                if (machine in test_mod_machines):
                    test_mod = item[1]

        tests.append(CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler, testmod=test_mod))

    if (inherits_from is not None):
        inherits_from = [inherits_from] if isinstance(inherits_from, six.string_types) else inherits_from
        for inherits in inherits_from:
            inherited_tests = get_test_suite(inherits, machine, compiler)

            expect(len(set(tests) & set(inherited_tests)) == 0,
                   "Tests {} defined in multiple suites".format(", ".join(set(tests) & set(inherited_tests))))
            tests.extend(inherited_tests)

    return tests

    def __init__(self, machine, compiler, cimeroot, caseroot, mpilib, debug=False):
        self._machine  = Machines(machine=machine)
        self._compiler = compiler
        self._cimeroot = cimeroot
        self._caseroot = caseroot
        self._mpilib   = mpilib
        self._debug    = debug

        self._module_system = self._machine.get_module_system_type()

def run_model(case):
###############################################################################

    # Set OMP_NUM_THREADS
    tm = TaskMaker(case)
    num_threads = tm.thread_count
    os.environ["OMP_NUM_THREADS"] = str(num_threads)

    # Run the model
    logger.info("%s MODEL EXECUTION BEGINS HERE" %(time.strftime("%Y-%m-%d %H:%M:%S")))

    machine = Machines(machine=case.get_value("MACH"))
    cmd = machine.get_full_mpirun(tm, case, "case.run")
    cmd = case.get_resolved_value(cmd)

    logger.info("run command is %s " %cmd)
    rundir = case.get_value("RUNDIR")
    run_cmd_no_fail(cmd, from_dir=rundir)
    logger.info( "%s MODEL EXECUTION HAS FINISHED" %(time.strftime("%Y-%m-%d %H:%M:%S")))

def find_all_supported_platforms():
###############################################################################
    """
    Returns a set of all ACME supported platforms as defined in the
    XML configuration file config_machines.xml in the ACME source
    tree. A platform is defined by a triple (machine name, compiler,
    mpi library).
    """
    machines = CIME.utils.get_machines()
    machobj = Machines(machine=machine)
    platform_set = set()

    for machine in machines:
        machobj.set_machine(machine)
        compilers, mpilibs = machobj.get_value("COMPILERS"), machobj.get_value("MPILIBS")
        for compiler in compilers:
            for mpilib in mpilibs:
                platform_set.add((machine, compiler, mpilib))

    return list(platform_set)

def get_test_suite(suite, machine=None, compiler=None, skip_inherit=False):
###############################################################################
    """
    Return a list of FULL test names for a suite.
    """
    expect(suite in get_test_suites(), "Unknown test suite: '{}'".format(suite))
    machobj = Machines(machine=machine)
    machine = machobj.get_machine_name()

    if(compiler is None):
        compiler = machobj.get_default_compiler()
    expect(machobj.is_valid_compiler(compiler),"Compiler {} not valid for machine {}".format(compiler,machine))

    inherits_from, _, _, tests_raw = get_test_data(suite)
    tests = []
    for item in tests_raw:
        expect(isinstance(item, six.string_types), "Bad type of test {}, expected string".format(item))

        test_mod = None
        test_components = item.split(".")
        expect(len(test_components) in [3, 4], "Bad test name {}".format(item))

        if (len(test_components) == 4):
            test_name = ".".join(test_components[:-1])
            test_mod = test_components[-1]
        else:
            test_name = item

        tests.append(CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler, testmod=test_mod))

    if not skip_inherit:
        for inherits in inherits_from:
            inherited_tests = get_test_suite(inherits, machine, compiler)

            for inherited_test in inherited_tests:
                if inherited_test not in tests:
                    tests.append(inherited_test)

    return tests

def _main():
    output, build_dir, build_optimized, clean,\
        cmake_args, compiler, enable_genf90, machine, machines_dir,\
        make_j, use_mpi, mpilib, mpirun_command, test_spec_dir, ctest_args,\
        use_openmp, xml_test_list, verbose \
        = parse_command_line(sys.argv)

#=================================================
# Find directory and file paths.
#=================================================
    suite_specs = []
    # TODO: this violates cime policy of direct access to xml
    # should be moved to CIME/XML
    if xml_test_list is not None:
        test_xml_tree = ElementTree()
        test_xml_tree.parse(xml_test_list)
        known_paths = {
            "here": os.path.abspath(os.path.dirname(xml_test_list)),
            }
        suite_specs.extend(suites_from_xml(test_xml_tree, known_paths))
    if test_spec_dir is not None:
        suite_specs.append(
            TestSuiteSpec("__command_line_test__",
                          ["__command_line_test__"],
                          [os.path.abspath(test_spec_dir)])
            )


    if machines_dir is not None:
        machines_file = os.path.join(machines_dir, "config_machines.xml")
        machobj = Machines(infile=machines_file, machine=machine)
    else:
        machobj = Machines(machine=machine)

    # Create build directory if necessary.
    build_dir = os.path.abspath(build_dir)

    if not os.path.isdir(build_dir):
        os.mkdir(build_dir)

    # Switch to the build directory.
    os.chdir(build_dir)

    #=================================================
    # Functions to perform various stages of build.
    #=================================================

    if not use_mpi:
        mpilib = "mpi-serial"
    elif mpilib is None:
        mpilib = machobj.get_default_MPIlib()
        logger.info("Using mpilib: {}".format(mpilib))

    if compiler is None:
        compiler = machobj.get_default_compiler()
        logger.info("Compiler is {}".format(compiler))

    compilerobj = Compilers(machobj, compiler=compiler, mpilib=mpilib)

    pfunit_path = find_pfunit(compilerobj, mpilib=mpilib, use_openmp=use_openmp)

    debug = not build_optimized
    os_ = machobj.get_value("OS")

    # Create the environment, and the Macros.cmake file
    #
    #
    configure(machobj, build_dir, ["CMake"], compiler, mpilib, debug, os_,
              unit_testing=True)
    machspecific = EnvMachSpecific(build_dir, unit_testing=True)

    fake_case = FakeCase(compiler, mpilib, debug)
    machspecific.load_env(fake_case)
    os.environ["OS"] = os_
    os.environ["COMPILER"] = compiler
    os.environ["DEBUG"] = stringify_bool(debug)
    os.environ["MPILIB"] = mpilib
    if use_openmp:
        os.environ["compile_threaded"] = "true"
    else:
        os.environ["compile_threaded"] = "false"

    os.environ["UNIT_TEST_HOST"] = socket.gethostname()
    if "NETCDF_PATH" in os.environ and not "NETCDF" in os.environ:
        # The CMake Netcdf find utility that we use (from pio2) seems to key off
        # of the environment variable NETCDF, but not NETCDF_PATH
        logger.info("Setting NETCDF environment variable: {}".format(os.environ["NETCDF_PATH"]))
        os.environ["NETCDF"] = os.environ["NETCDF_PATH"]

    if not use_mpi:
        mpirun_command = ""
    elif mpirun_command is None:
        mpi_attribs = {
            "compiler" : compiler,
            "mpilib"   : mpilib,
            "threaded" : use_openmp,
            "unit_testing" : True
        }

        # We can get away with specifying case=None since we're using exe_only=True
#.........这里部分代码省略.........

    def configure(self, compset_name, grid_name, machine_name=None,
                  project=None, pecount=None, compiler=None, mpilib=None,
                  user_compset=False, pesfile=None,
                  user_grid=False, gridfile=None, ninst=1, test=False,
                  walltime=None, queue=None):

        #--------------------------------------------
        # compset, pesfile, and compset components
        #--------------------------------------------
        self._set_compset_and_pesfile(compset_name, user_compset=user_compset, pesfile=pesfile)

        self._components = self.get_compset_components()
        #FIXME - if --user-compset is True then need to determine that
        #all of the compset settings are valid

        #--------------------------------------------
        # grid
        #--------------------------------------------
        if user_grid is True and gridfile is not None:
            self.set_value("GRIDS_SPEC_FILE", gridfile)
        grids = Grids(gridfile)

        gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname)

        self._gridname = gridinfo["GRID"]
        for key,value in gridinfo.items():
            logger.debug("Set grid %s %s"%(key,value))
            self.set_lookup_value(key,value)

        #--------------------------------------------
        # component config data
        #--------------------------------------------
        self._get_component_config_data()

        self.get_compset_var_settings()

        #--------------------------------------------
        # machine
        #--------------------------------------------
        # set machine values in env_xxx files
        machobj = Machines(machine=machine_name)
        machine_name = machobj.get_machine_name()
        self.set_value("MACH",machine_name)
        nodenames = machobj.get_node_names()
        nodenames =  [x for x in nodenames if
                      '_system' not in x and '_variables' not in x and 'mpirun' not in x and\
                      'COMPILER' not in x and 'MPILIB' not in x]

        for nodename in nodenames:
            value = machobj.get_value(nodename, resolved=False)
            type_str = self.get_type_info(nodename)
            if type_str is not None:
                logger.debug("machine nodname %s value %s"%(nodename, value))
                self.set_value(nodename, convert_to_type(value, type_str, nodename))

        if compiler is None:
            compiler = machobj.get_default_compiler()
        else:
            expect(machobj.is_valid_compiler(compiler),
                   "compiler %s is not supported on machine %s" %(compiler, machine_name))

        self.set_value("COMPILER",compiler)

        if mpilib is None:
            mpilib = machobj.get_default_MPIlib({"compiler":compiler})
        else:
            expect(machobj.is_valid_MPIlib(mpilib, {"compiler":compiler}),
                   "MPIlib %s is not supported on machine %s" %(mpilib, machine_name))
        self.set_value("MPILIB",mpilib)

        machdir = machobj.get_machines_dir()
        self.set_value("MACHDIR", machdir)

        # Create env_mach_specific settings from machine info.
        env_mach_specific_obj = self.get_env("mach_specific")
        env_mach_specific_obj.populate(machobj)
        self.schedule_rewrite(env_mach_specific_obj)

        #--------------------------------------------
        # pe payout
        #--------------------------------------------
        match1 = re.match('([0-9]+)x([0-9]+)', "" if pecount is None else pecount)
        match2 = re.match('([0-9]+)', "" if pecount is None else pecount)
        pes_ntasks = {}
        pes_nthrds = {}
        pes_rootpe = {}
        if match1:
            opti_tasks = match1.group(1)
            opti_thrds = match1.group(2)
        elif match2:
            opti_tasks = match2.group(1)
            opti_thrds = 1

        other = {}
        if match1 or match2:
            for component_class in self._component_classes:
                if component_class == "DRV":
                    component_class = "CPL"
                string = "NTASKS_" + component_class
                pes_ntasks[string] = opti_tasks
#.........这里部分代码省略.........

def load_balancing_submit(compset, res, pesfile, mpilib, compiler, project, machine,
                          extra_options_file, test_id, force_purge, test_root):
################################################################################
    # Read in list of pes from given file
    expect(os.access(pesfile, os.R_OK), 'ERROR: File %s not found', pesfile)

    logger.info('Reading XML file %s. Searching for pesize entries:', pesfile)
    try:
        pesobj = Pes(pesfile)
    except ParseError:
        expect(False, 'ERROR: File %s not parseable', pesfile)

    pesize_list = []
    grid_nodes = pesobj.get_children("grid")
    for gnode in grid_nodes:
        mach_nodes = pesobj.get_children("mach", root=gnode)
        for mnode in mach_nodes:
            pes_nodes = pesobj.get_children("pes", root=mnode)
            for pnode in pes_nodes:
                pesize = pesobj.get(pnode, 'pesize')
                if not pesize:
                    logger.critical('No pesize for pes node in file %s', pesfile)
                if pesize in pesize_list:
                    logger.critical('pesize %s duplicated in file %s', pesize, pesfile)
                pesize_list.append(pesize)

    expect(pesize_list, 'ERROR: No grid entries found in pes file {}'.format(pesfile))

    machobj = Machines(machine=machine)
    if test_root is None:
        test_root = machobj.get_value("CIME_OUTPUT_ROOT")
    if machine is None:
        machine = machobj.get_machine_name()
        print "machine is {}".format(machine)
    if compiler is None:
        compiler = machobj.get_default_compiler()
        print "compiler is {}".format(compiler)
    if mpilib is None:
        mpilib = machobj.get_default_MPIlib({"compiler":compiler})




    test_names = []
    for i in xrange(len(pesize_list)):
        test_names.append(get_full_test_name("PFS_I{}".format(i),grid=res, compset=compset,
                                             machine=machine, compiler=compiler))
        casedir = os.path.join(test_root, test_names[-1] + "." + test_id)
        print "casedir is {}".format(casedir)
        if os.path.isdir(casedir):
            if force_purge:
                logger.info('Removing directory %s', casedir)
                shutil.rmtree(casedir)
            else:
                expect(False,
                       "casedir {} already exists, use the --force-purge option, --test-root or"
                       " --test-id options".format(casedir))

    tests = TestScheduler(test_names, no_setup = True,
                          compiler=compiler, machine_name=machine, mpilib=mpilib,
                          test_root=test_root, test_id=test_id, project=project)
    success = tests.run_tests(wait=True)
    expect(success, "Error in creating cases")
    testnames = []
    for test in tests.get_testnames():
        testname =  os.path.join(test_root, test + "." + test_id)
        testnames.append( testname)
        logger.info("test is {}".format(testname))
        with Case(testname) as case:
            pes_ntasks, pes_nthrds, pes_rootpe, _, _, _ = \
                                                    pesobj.find_pes_layout('any', 'any', 'any', pesize_opts=pesize_list.pop(0))
            for key in pes_ntasks:
                case.set_value(key, pes_ntasks[key])
            for key in pes_nthrds:
                case.set_value(key, pes_nthrds[key])
            for key in pes_rootpe:
                case.set_value(key, pes_rootpe[key])

            if extra_options_file is not None:
                try:
                    extras = open(extra_options_file, 'r')
                    for line in extras.readlines():
                        split = line.split('=')
                        if len(split) == 2:
                            logger.info('setting %s=%s', split[0], split[1])
                            case.set_value(split[0], split[1])
                        else:
                            logger.debug('ignoring line in {}: {}'.format(
                                extra_options_file, line))
                    extras.close()
                except IOError:
                    expect(False, "ERROR: Could not read file {}".format(extra_options_file))


    tests = TestScheduler(test_names, use_existing=True, test_root=test_root, test_id=test_id)
    success = tests.run_tests(wait=False)
    expect(success, "Error in running cases")

    # need to fix
    logger.info('Timing jobs submitted. After jobs completed, run to optimize '
#.........这里部分代码省略.........

    def configure(self, compset_name, grid_name, machine_name=None,
                  project=None, pecount=None, compiler=None, mpilib=None,
                  user_compset=False, pesfile=None,
                  user_grid=False, gridfile=None, ninst=1, test=False):

        #--------------------------------------------
        # compset, pesfile, and compset components
        #--------------------------------------------
        self._set_compset_and_pesfile(compset_name, user_compset=user_compset, pesfile=pesfile)

        self._components = self.get_compset_components()
        #FIXME - if --user-compset is True then need to determine that
        #all of the compset settings are valid

        #--------------------------------------------
        # grid
        #--------------------------------------------
        if user_grid is True and gridfile is not None:
            self.set_value("GRIDS_SPEC_FILE", gridfile);
        grids = Grids(gridfile)

        gridinfo = grids.get_grid_info(name=grid_name, compset=self._compsetname)
        self._gridname = gridinfo["GRID"]
        for key,value in gridinfo.items():
            logger.debug("Set grid %s %s"%(key,value))
            self.set_value(key,value)

        #--------------------------------------------
        # component config data
        #--------------------------------------------
        self._get_component_config_data()

        self.get_compset_var_settings()

        # Add the group and elements for the config_files.xml
        for idx, config_file in enumerate(self._component_config_files):
            self.set_value(config_file[0],config_file[1])

        #--------------------------------------------
        # machine
        #--------------------------------------------
        # set machine values in env_xxx files
        machobj = Machines(machine=machine_name)
        machine_name = machobj.get_machine_name()
        self.set_value("MACH",machine_name)
        nodenames = machobj.get_node_names()
        nodenames =  [x for x in nodenames if
                      '_system' not in x and '_variables' not in x and 'mpirun' not in x and\
                      'COMPILER' not in x and 'MPILIB' not in x]

        for nodename in nodenames:
            value = machobj.get_value(nodename)
            type_str = self.get_type_info(nodename)
            if type_str is not None:
                self.set_value(nodename, convert_to_type(value, type_str, nodename))

        if compiler is None:
            compiler = machobj.get_default_compiler()
        else:
            expect(machobj.is_valid_compiler(compiler),
                   "compiler %s is not supported on machine %s" %(compiler, machine_name))

        self.set_value("COMPILER",compiler)

        if mpilib is None:
            mpilib = machobj.get_default_MPIlib({"compiler":compiler})
        else:
            expect(machobj.is_valid_MPIlib(mpilib, {"compiler":compiler}),
                   "MPIlib %s is not supported on machine %s" %(mpilib, machine_name))
        self.set_value("MPILIB",mpilib)

        machdir = machobj.get_machines_dir()
        self.set_value("MACHDIR", machdir)

        # Overwriting an existing exeroot or rundir can cause problems
        exeroot = self.get_value("EXEROOT")
        rundir = self.get_value("RUNDIR")
        for wdir in (exeroot, rundir):
            if os.path.exists(wdir):
                expect(not test, "Directory %s already exists, aborting test"% wdir)
                response = raw_input("\nDirectory %s already exists, (r)eplace, (a)bort, or (u)se existing?"% wdir)
                if response.startswith("r"):
                    shutil.rmtree(wdir)
                else:
                    expect(response.startswith("u"), "Aborting by user request")

        # the following go into the env_mach_specific file
        vars = ("module_system", "environment_variables", "mpirun")
        env_mach_specific_obj = self._get_env("mach_specific")
        for var in vars:
            nodes = machobj.get_first_child_nodes(var)
            for node in nodes:
                env_mach_specific_obj.add_child(node)

        #--------------------------------------------
        # pe payout
        #--------------------------------------------
        pesobj = Pes(self._pesfile)

        #FIXME - add pesize_opts as optional argument below
#.........这里部分代码省略.........

    def __init__(self, test_names, test_data=None,
                 no_run=False, no_build=False, no_setup=False, no_batch=None,
                 test_root=None, test_id=None,
                 machine_name=None, compiler=None,
                 baseline_root=None, baseline_cmp_name=None, baseline_gen_name=None,
                 clean=False, namelists_only=False,
                 project=None, parallel_jobs=None,
                 walltime=None, proc_pool=None,
                 use_existing=False, save_timing=False, queue=None,
                 allow_baseline_overwrite=False, output_root=None,
                 force_procs=None, force_threads=None, mpilib=None,
                 input_dir=None, pesfile=None, mail_user=None, mail_type=None):
    ###########################################################################
        self._cime_root       = CIME.utils.get_cime_root()
        self._cime_model      = get_model()
        self._save_timing     = save_timing
        self._queue           = queue
        self._test_data       = {} if test_data is None else test_data # Format:  {test_name -> {data_name -> data}}
        self._mpilib          = mpilib  # allow override of default mpilib
        self._completed_tests = 0
        self._input_dir       = input_dir
        self._pesfile         = pesfile
        self._allow_baseline_overwrite = allow_baseline_overwrite

        self._mail_user = mail_user
        self._mail_type = mail_type

        self._machobj = Machines(machine=machine_name)

        self._model_build_cost = 4

        # If user is forcing procs or threads, re-write test names to reflect this.
        if force_procs or force_threads:
            test_names = _translate_test_names_for_new_pecount(test_names, force_procs, force_threads)

        self._no_setup = no_setup
        self._no_build = no_build or no_setup or namelists_only
        self._no_run   = no_run or self._no_build
        self._output_root = output_root
        # Figure out what project to use
        if project is None:
            self._project = CIME.utils.get_project()
            if self._project is None:
                self._project = self._machobj.get_value("PROJECT")
        else:
            self._project = project

        # We will not use batch system if user asked for no_batch or if current
        # machine is not a batch machine
        self._no_batch = no_batch or not self._machobj.has_batch_system()
        expect(not (self._no_batch and self._queue is not None),
               "Does not make sense to request a queue without batch system")

        # Determine and resolve test_root
        if test_root is not None:
            self._test_root = test_root
        elif self._output_root is not None:
            self._test_root = self._output_root
        else:
            self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT")

        if self._project is not None:
            self._test_root = self._test_root.replace("$PROJECT", self._project)

        self._test_root = os.path.abspath(self._test_root)
        self._test_id   = test_id if test_id is not None else CIME.utils.get_timestamp()

        self._compiler = self._machobj.get_default_compiler() if compiler is None else compiler

        self._clean          = clean
        self._namelists_only = namelists_only

        self._walltime = walltime

        if parallel_jobs is None:
            self._parallel_jobs = min(len(test_names),
                                      self._machobj.get_value("MAX_MPITASKS_PER_NODE"))
        else:
            self._parallel_jobs = parallel_jobs

        self._baseline_cmp_name = baseline_cmp_name # Implies comparison should be done if not None
        self._baseline_gen_name = baseline_gen_name # Implies generation should be done if not None

        # Compute baseline_root
        self._baseline_root = baseline_root if baseline_root is not None \
                              else self._machobj.get_value("BASELINE_ROOT")

        if self._project is not None:
            self._baseline_root = self._baseline_root.replace("$PROJECT", self._project)

        self._baseline_root = os.path.abspath(self._baseline_root)

        if baseline_cmp_name or baseline_gen_name:
            if self._baseline_cmp_name:
                full_baseline_dir = os.path.join(self._baseline_root, self._baseline_cmp_name)
                expect(os.path.isdir(full_baseline_dir),
                       "Missing baseline comparison directory {}".format(full_baseline_dir))

            # the following is to assure that the existing generate directory is not overwritten
            if self._baseline_gen_name:
#.........这里部分代码省略.........

    def __init__(self, test_names,
                 no_run=False, no_build=False, no_batch=None,
                 test_root=None, test_id=None,
                 machine_name=None,compiler=None,
                 baseline_root=None, baseline_name=None,
                 clean=False,compare=False, generate=False, namelists_only=False,
                 project=None, parallel_jobs=None,
                 xml_machine=None, xml_compiler=None, xml_category=None,xml_testlist=None):
    ###########################################################################
        self._cime_root = CIME.utils.get_cime_root()
        self._cime_model = CIME.utils.get_model()
        # needed for perl interface
        os.environ["CIMEROOT"] = self._cime_root
        self._machobj   = Machines(machine=machine_name)
        machine_name    = self._machobj.get_machine_name()

        self._no_build = no_build      if not namelists_only else True
        self._no_run   = no_run        if not self._no_build else True

        # Figure out what project to use
        if (project is None):
            self._project = CIME.utils.get_project()
            if (self._project is None):
                self._project = self._machobj.get_value("PROJECT")
        else:
            self._project = project

        # We will not use batch system if user asked for no_batch or if current
        # machine is not a batch machine
        self._no_batch = no_batch or not self._machobj.has_batch_system()

        self._test_root = test_root if test_root is not None else self._machobj.get_value("CESMSCRATCHROOT")
        if (self._project is not None):
            self._test_root = self._test_root.replace("$PROJECT", self._project)
        self._test_root = os.path.abspath(self._test_root)

        self._test_id = test_id if test_id is not None else CIME.utils.get_utc_timestamp()

        self._compiler = compiler if compiler is not None else self._machobj.get_default_compiler()
        expect(self._machobj.is_valid_compiler(self._compiler),
               "Compiler %s not valid for machine %s" % (self._compiler,machine_name))

        self._clean          = clean
        self._namelists_only = namelists_only

        # Extra data associated with tests, do not modify after construction
        # test_name -> test_data
        #   test_data: name -> value
        self._test_data      = {}

        # If xml options are provided get tests from xml file, otherwise use acme dictionary
        if(not test_names and (xml_machine is not None or xml_category is not None or xml_compiler is not None or xml_testlist is not None)):
            test_data = CIME.test_utils.get_tests_from_xml(xml_machine, xml_category, xml_compiler, xml_testlist, machine_name, compiler)
            test_names = [item["name"] for item in test_data]
            for test_datum in test_data:
                self._test_data[test_datum["name"]] = test_datum
        else:
            expect(len(test_names) > 0, "No tests to run")
            test_names = update_acme_tests.get_full_test_names(test_names, machine_name, self._compiler)

        if (parallel_jobs is None):
            self._parallel_jobs  = min(len(test_names), int(self._machobj.get_value("MAX_TASKS_PER_NODE")))
        else:
            self._parallel_jobs = parallel_jobs

        self._baseline_cmp_name = None
        self._baseline_gen_name = None
        self._compare = False
        self._generate = False
        if (compare or generate):
            # Figure out what baseline name to use
            if (baseline_name is None):
                if(compare is not None and isinstance(compare,str)):
                    self._baseline_cmp_name = compare
                    self._compare = True
                if(generate is not None and isinstance(generate,str)):
                    self._baseline_gen_name = generate
                    self._generate = True
                branch_name = CIME.utils.get_current_branch(repo=self._cime_root)
                expect(branch_name is not None, "Could not determine baseline name from branch, please use -b option")
                if(self._compare and self._baseline_cmp_name is None):
                    self._baseline_cmp_name = os.path.join(self._compiler, branch_name)
                if(self._generate and self._baseline_gen_name is None):
                    self._baseline_gen_name = os.path.join(self._compiler, branch_name)
            else:
                if(compare):
                    self._compare = True
                    self._baseline_cmp_name  = baseline_name
                    if (not self._baseline_cmp_name.startswith("%s/" % self._compiler)):
                        self._baseline_cmp_name = os.path.join(self._compiler, self._baseline_cmp_name)
                if(generate):
                    self._generate = True
                    self._baseline_gen_name  = baseline_name
                    if (not self._baseline_gen_name.startswith("%s/" % self._compiler)):
                        self._baseline_gen_name = os.path.join(self._compiler, self._baseline_gen_name)

            # Compute baseline_root
            self._baseline_root = baseline_root if baseline_root is not None else self._machobj.get_value("CCSM_BASELINE")
            if (self._project is not None):
                self._baseline_root = self._baseline_root.replace("$PROJECT", self._project)
#.........这里部分代码省略.........

class SystemTest(object):
###############################################################################

    ###########################################################################
    def __init__(self, test_names,
                 no_run=False, no_build=False, no_batch=None,
                 test_root=None, test_id=None,
                 machine_name=None,compiler=None,
                 baseline_root=None, baseline_name=None,
                 clean=False,compare=False, generate=False, namelists_only=False,
                 project=None, parallel_jobs=None,
                 xml_machine=None, xml_compiler=None, xml_category=None,xml_testlist=None):
    ###########################################################################
        self._cime_root = CIME.utils.get_cime_root()
        self._cime_model = CIME.utils.get_model()
        # needed for perl interface
        os.environ["CIMEROOT"] = self._cime_root
        self._machobj   = Machines(machine=machine_name)
        machine_name    = self._machobj.get_machine_name()

        self._no_build = no_build      if not namelists_only else True
        self._no_run   = no_run        if not self._no_build else True

        # Figure out what project to use
        if (project is None):
            self._project = CIME.utils.get_project()
            if (self._project is None):
                self._project = self._machobj.get_value("PROJECT")
        else:
            self._project = project

        # We will not use batch system if user asked for no_batch or if current
        # machine is not a batch machine
        self._no_batch = no_batch or not self._machobj.has_batch_system()

        self._test_root = test_root if test_root is not None else self._machobj.get_value("CESMSCRATCHROOT")
        if (self._project is not None):
            self._test_root = self._test_root.replace("$PROJECT", self._project)
        self._test_root = os.path.abspath(self._test_root)

        self._test_id = test_id if test_id is not None else CIME.utils.get_utc_timestamp()

        self._compiler = compiler if compiler is not None else self._machobj.get_default_compiler()
        expect(self._machobj.is_valid_compiler(self._compiler),
               "Compiler %s not valid for machine %s" % (self._compiler,machine_name))

        self._clean          = clean
        self._namelists_only = namelists_only

        # Extra data associated with tests, do not modify after construction
        # test_name -> test_data
        #   test_data: name -> value
        self._test_data      = {}

        # If xml options are provided get tests from xml file, otherwise use acme dictionary
        if(not test_names and (xml_machine is not None or xml_category is not None or xml_compiler is not None or xml_testlist is not None)):
            test_data = CIME.test_utils.get_tests_from_xml(xml_machine, xml_category, xml_compiler, xml_testlist, machine_name, compiler)
            test_names = [item["name"] for item in test_data]
            for test_datum in test_data:
                self._test_data[test_datum["name"]] = test_datum
        else:
            expect(len(test_names) > 0, "No tests to run")
            test_names = update_acme_tests.get_full_test_names(test_names, machine_name, self._compiler)

        if (parallel_jobs is None):
            self._parallel_jobs  = min(len(test_names), int(self._machobj.get_value("MAX_TASKS_PER_NODE")))
        else:
            self._parallel_jobs = parallel_jobs

        self._baseline_cmp_name = None
        self._baseline_gen_name = None
        self._compare = False
        self._generate = False
        if (compare or generate):
            # Figure out what baseline name to use
            if (baseline_name is None):
                if(compare is not None and isinstance(compare,str)):
                    self._baseline_cmp_name = compare
                    self._compare = True
                if(generate is not None and isinstance(generate,str)):
                    self._baseline_gen_name = generate
                    self._generate = True
                branch_name = CIME.utils.get_current_branch(repo=self._cime_root)
                expect(branch_name is not None, "Could not determine baseline name from branch, please use -b option")
                if(self._compare and self._baseline_cmp_name is None):
                    self._baseline_cmp_name = os.path.join(self._compiler, branch_name)
                if(self._generate and self._baseline_gen_name is None):
                    self._baseline_gen_name = os.path.join(self._compiler, branch_name)
            else:
                if(compare):
                    self._compare = True
                    self._baseline_cmp_name  = baseline_name
                    if (not self._baseline_cmp_name.startswith("%s/" % self._compiler)):
                        self._baseline_cmp_name = os.path.join(self._compiler, self._baseline_cmp_name)
                if(generate):
                    self._generate = True
                    self._baseline_gen_name  = baseline_name
                    if (not self._baseline_gen_name.startswith("%s/" % self._compiler)):
                        self._baseline_gen_name = os.path.join(self._compiler, self._baseline_gen_name)

#.........这里部分代码省略.........

class EnvModule(object):

    # TODO - write env_mach_specific files into case

    # Public API

    def __init__(self, machine, compiler, cimeroot, caseroot, mpilib, debug=False):
        self._machine  = Machines(machine=machine)
        self._compiler = compiler
        self._cimeroot = cimeroot
        self._caseroot = caseroot
        self._mpilib   = mpilib
        self._debug    = debug

        self._module_system = self._machine.get_module_system_type()

    def load_env_for_case(self):
        mach_specific = EnvMachSpecific(caseroot=self._caseroot)

        module_nodes = mach_specific.get_node("modules")
        env_nodes    = mach_specific.get_node("environment_variables")

        if (module_nodes is not None):
            modules_to_load = self._compute_module_actions(module_nodes)
            self.load_modules(modules_to_load)
        if (env_nodes is not None):
            envs_to_set = self._compute_env_actions(env_nodes)
            self.load_envs(envs_to_set)

    def load_modules(self, modules_to_load):
        if (self._module_system == "module"):
            self._load_module_modules(modules_to_load)
        elif (self._module_system == "soft"):
            self._load_soft_modules(modules_to_load)
        elif (self._module_system == "dotkit"):
            self._load_dotkit_modules(modules_to_load)
        elif (self._module_system == "none"):
            self._load_none_modules(modules_to_load)
        else:
            expect(False, "Unhandled module system '%s'" % self._module_system)

    def load_envs(self, envs_to_set):
        for env_name, env_value in envs_to_set:
            # Let bash do the work on evaluating and resolving env_value
            os.environ[env_name] = run_cmd("echo %s" % env_value)

    # Private API

    def _compute_module_actions(self, module_nodes):
        return self._compute_actions(module_nodes, "command")

    def _compute_env_actions(self, env_nodes):
        return self._compute_actions(env_nodes, "env")

    def _compute_actions(self, nodes, child_tag):
        result = [] # list of tuples ("name", "argument")

        for node in nodes:
            if (self._match_attribs(node.attrib)):
                for child in node:
                    expect(child.tag == child_tag, "Expected %s element" % child_tag)
                    result.append( (child.get("name"), child.text) )

        return result

    def _match_attribs(self, attribs):
        if ("compiler" in attribs and
            not self._match(self._compiler, attribs["compiler"])):
            return False
        elif ("mpilib" in attribs and
            not self._match(self._mpilib, attribs["mpilib"])):
            return False
        elif ("debug" in attribs and
            not self._match("TRUE" if self._debug else "FALSE", attribs["debug"].upper())):
            return False

        return True

    def _match(self, my_value, xml_value):
        if (xml_value.startswith("!")):
            return my_value != xml_value[1:]
        else:
            return my_value == xml_value

    def _load_module_modules(self, modules_to_load):
        python_mod_cmd = self._machine.get_module_system_cmd_path("python")
        for action, argument in modules_to_load:
            cmd = "%s %s %s" % (python_mod_cmd, action, argument)
            py_module_code = run_cmd(cmd)
            exec(py_module_code)

    def _load_soft_modules(self, modules_to_load):
        expect(False, "Not yet implemented")

    def _load_dotkit_modules(self, modules_to_load):
        expect(False, "Not yet implemented")

    def _load_none_modules(self, modules_to_load):
        expect(False, "Not yet implemented")

def parse_command_line(args, description):
###############################################################################
    help_str = """
    Solve a Mixed Integer Linear Program to find a PE layout that minimizes
    the wall-clock time per model day.
    """
    parser = argparse.ArgumentParser(usage=help_str,
                                     description=description,
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    CIME.utils.setup_standard_logging_options(parser)

    parser.add_argument('--test-id', default=DEFAULT_TESTID,
                        help='test-id to use for all timing runs')

    parser.add_argument("-r", "--test-root",
                        help="Where test cases were created."
                        " Will default to output root as defined in the config_machines file")

    parser.add_argument('--timing-dir', help='alternative to using casename '
                        'to find timing data, instead read all files in'
                        ' this directory')

    parser.add_argument('--blocksize',
                        help='default minimum size of blocks to assign to all '
                        'components. Components can be assigned different '
                        'blocksizes using --blocksize_XXX. Default 1', type=int)

    for c in COMPONENT_LIST:
        parser.add_argument('--blocksize-%s' % c.lower(),
                            help='minimum blocksize for component %s, if '
                            'different from --blocksize', type=int)

    parser.add_argument('--total-tasks', type=int,
                        help='Number of pes available for assignment')

    parser.add_argument("--layout",
                        help="name of layout to solve (default selected internally)")

    parser.add_argument("--graph-models", action="store_true",
                        help="plot cost v. ntasks models. requires matplotlib")

    parser.add_argument("--print-models", action="store_true",
                        help="print all costs and ntasks")

    parser.add_argument("--pe-output", help="write pe layout to file")

    parser.add_argument('--json-output', help="write MILP data to .json file")

    parser.add_argument('--json-input', help="solve using data from .json file")

    args = CIME.utils.parse_args_and_handle_standard_logging_options(args,
                                                                     parser)
    if args.total_tasks is None and args.json_input is None:
        expect(args.total_tasks is not None or args.json_input is not None,
               "--total-tasks or --json-input option must be set")

    blocksizes = {}
    for c in COMPONENT_LIST:
        attrib = 'blocksize_%s' % c.lower()
        if getattr(args, attrib) is not None:
            blocksizes[c] = getattr(args, attrib)
        elif args.blocksize is not None:
            blocksizes[c] = args.blocksize
    test_root = args.test_root
    if test_root is None:
        machobj = Machines()
        test_root = machobj.get_value("CIME_OUTPUT_ROOT")

    return (args.test_id, test_root, args.timing_dir, blocksizes,
            args.total_tasks, args.layout, args.graph_models,
            args.print_models, args.pe_output, args.json_output,
            args.json_input)