Python files.Files类代码示例

    def _create_newcase_phase(self, test):
    ###########################################################################
        test_dir = self._get_test_dir(test)

        test_case, case_opts, grid, compset, machine, compiler, test_mods = CIME.utils.parse_test_name(test)
        if (compiler != self._compiler):
            raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test, self._compiler))
        if (self._parallel_jobs == 1):
            scratch_dir = self._machobj.get_value("CESMSCRATCHROOT")
            if (self._project is not None):
                scratch_dir = scratch_dir.replace("$PROJECT", self._project)
            sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
        else:
            # Parallelizing builds introduces potential sync problems with sharedlibroot
            # Just let every case build it's own
            sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
        create_newcase_cmd = "%s -model %s -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -sharedlibroot %s" % \
                              (os.path.join(self._cime_root,"scripts", "create_newcase"),
                               self._cime_model,test_dir, grid, machine, compiler, compset, test_case, self._project,
                               sharedlibroot)
        if (test_case != 'PFS'):
            create_newcase_cmd += " -nosavetiming "
        if (case_opts is not None):
            create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
        if (test_mods is not None):
            files = Files()
            (component, mods) = test_mods.split('/')
            testmods_dir = files.get_value("TESTS_MODS_DIR",{"component": component})
            test_mod_file = os.path.join(testmods_dir, component, mods)
            if (not os.path.exists(test_mod_file)):
                self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
        logging.info("Calling create_newcase: "+create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)

    def __init__(self, infile=None, files=None, machine=None):
        """
        initialize an object
        if a filename is provided it will be used,
        otherwise if a files object is provided it will be used
        otherwise create a files object from default values
        """

        self.machine_node = None
        self.machine = None
        self.machines_dir = None

        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("MACHINES_SPEC_FILE", resolved=False)
            infile = files.get_resolved_value(infile)

        self.machines_dir = os.path.dirname(infile)

        GenericXML.__init__(self, infile)

        # Append the contents of $HOME/.cime/config_machines.xml if it exists
        # This could cause problems if node matchs are repeated when only one is expected
        local_infile = os.path.join(os.environ.get("HOME"), ".cime", "config_machines.xml")
        logger.debug("Infile: %s", local_infile)
        if os.path.exists(local_infile):
            GenericXML.read(self, local_infile)

        if machine is None:
            machine = self.probe_machine_name()

        expect(machine is not None, "Could not initialize machine object from %s or %s" % (infile, local_infile))
        self.set_machine(machine)

    def __init__(self, compiler=None, machine=None, os_= None, mpilib=None, infile=None, files=None):
        """
        initialize an object
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("COMPILERS_SPEC_FILE")

        GenericXML.__init__(self, infile)

        self.machine        = machine
        self.os             = os_
        self.mpilib         = mpilib
        self.compiler_nodes = None # Listed from last to first
        self.compiler       = compiler

        if self.compiler is not None:
            self.set_compiler(compiler)

        #Append the contents of $HOME/.cime/config_compilers.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_compilers.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

    def setup(self, env_archive, components, files=None):
        if files is None:
            files = Files()

        components_node = env_archive.make_child("components", attributes={"version":"2.0"})

        model = get_model()
        if 'drv' not in components:
            components.append('drv')
        if 'dart' not in components and model == 'cesm':
            components.append('dart')

        for comp in components:
            infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp})

            if infile is not None and os.path.isfile(infile):
                arch = Archive(infile=infile, files=files)
                specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})
            else:
                if infile is None:
                    logger.debug("No archive file defined for component {}".format(comp))
                else:
                    logger.debug("Archive file {} for component {} not found".format(infile,comp))

                specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp})

            if specs is None:
                logger.debug("No archive specs found for component {}".format(comp))
            else:
                logger.debug("adding archive spec for {}".format(comp))
                env_archive.add_child(specs, root=components_node)

    def _create_caseroot_tools(self):
        machines_dir = os.path.abspath(self.get_value("MACHDIR"))
        toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools")
        # setup executable files in caseroot/
        exefiles = (os.path.join(toolsdir, "case.setup"),
                    os.path.join(toolsdir, "case.build"),
                    os.path.join(toolsdir, "case.submit"),
                    os.path.join(toolsdir, "preview_namelists"),
                    os.path.join(toolsdir, "check_input_data"),
                    os.path.join(toolsdir, "check_case"),
                    os.path.join(toolsdir, "archive_metadata.sh"),
                    os.path.join(toolsdir, "xmlchange"),
                    os.path.join(toolsdir, "xmlquery"))
        try:
            for exefile in exefiles:
                destfile = os.path.join(self._caseroot,os.path.basename(exefile))
                os.symlink(exefile, destfile)
        except Exception as e:
            logger.warning("FAILED to set up exefiles: %s" % str(e))

        # set up utility files in caseroot/Tools/
        toolfiles = (os.path.join(toolsdir, "check_lockedfiles"),
                     os.path.join(toolsdir, "lt_archive.sh"),
                     os.path.join(toolsdir, "getTiming"),
                     os.path.join(toolsdir, "save_provenance"),
                     os.path.join(machines_dir,"Makefile"),
                     os.path.join(machines_dir,"mkSrcfiles"),
                     os.path.join(machines_dir,"mkDepends"))

        for toolfile in toolfiles:
            destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile))
            expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile)
            try:
                os.symlink(toolfile, destfile)
            except Exception as e:
                logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile))

        # Create Macros file.
        machine = self.get_value("MACH")
        files = Files()
        # Use config_build if the environment variable is set, or if there is no
        # config_compilers file.
        if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \
           files.get_value("COMPILERS_SPEC_FILE") is None:
            build_file = files.get_value("BUILD_SPEC_FILE")
            machobj = Machines(machine=machine, files=files)
            macro_maker = Build(machobj)
            macros_path = os.path.join(self._caseroot, "Macros")
            with open(macros_path, "w") as macros_file:
                macro_maker.write_macros('Makefile', build_file, macros_file)

        # Copy any system or compiler Depends files to the case.
        compiler = self.get_value("COMPILER")
        for dep in (machine, compiler):
            dfile = "Depends.%s"%dep
            if os.path.isfile(os.path.join(machines_dir,dfile)):
                shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile))
        dfile = "Depends.%s.%s"%(machine,compiler)
        if os.path.isfile(os.path.join(machines_dir,dfile)):
            shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))

    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("PIO_SPEC_FILE")

        EntryID.__init__(self, infile)

def saveLogs(case, lid):
###############################################################################

    logdir = case.get_value("LOGDIR")
    if logdir is not None and len(logdir) > 0:
        if not os.path.isdir(logdir):
            os.makedirs(logdir)

        caseroot = case.get_value("CASEROOT")
        rundir = case.get_value("RUNDIR")

        # get components
        files = Files()
        config_file = files.get_value("CONFIG_DRV_FILE")
        component = Component(config_file)
        comps = [x.lower() for x in component.get_valid_model_components()]
        comps = [x.replace('drv', 'cpl') for x in comps]
        model = [case.get_value("MODEL")]
        comps = comps + model

        # for each component, compress log files and copy to logdir
        for comp in comps:
            logfile = os.path.join(rundir, comp + '.log.' + lid)
            if os.path.isfile(logfile):
                f_in = open(logfile)
                f_out = gzip.open(logfile + '.gz', 'wb')
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                os.remove(logfile)
                logfile_copy = logfile + '.gz'
                shutil.copy(logfile_copy,
                            os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))

    def __init__(self, batch_system=None, machine=None, infile=None, files=None):
        """
        initialize an object
        """
        if files is None:
            files = Files()
        if infile is None:
            infile = files.get_value("BATCH_SPEC_FILE")

        schema = files.get_schema("BATCH_SPEC_FILE")

        GenericXML.__init__(self, infile, schema=schema)

        self.batch_system_node = None
        self.machine_node      = None
        self.batch_system      = batch_system
        self.machine           = machine

        #Append the contents of $HOME/.cime/config_batch.xml if it exists
        #This could cause problems if node matchs are repeated when only one is expected
        infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml")
        if os.path.exists(infile):
            GenericXML.read(self, infile)

        if self.batch_system is not None:
            self.set_batch_system(self.batch_system, machine=machine)

    def __init__(self, infile=None, files=None):
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("GRIDS_SPEC_FILE")
        logger.debug(" Grid specification file is %s" % infile)

        GenericXML.__init__(self, infile)

    def __init__(self, infile=None):
        """
        initialize an object
        """
        if infile is None:
            files = Files()
            infile = files.get_value("CONFIG_DRV_FILE")

        EntryID.__init__(self,infile)

 def __init__(self, infile, files=None):
     """
     initialize a files object given input pes specification file
     """
     if files is None:
         files = Files()
     schema = files.get_schema("PES_SPEC_FILE")
     logger.debug("DEBUG: infile is {}".format(infile))
     GenericXML.__init__(self, infile, schema=schema)

 def __init__(self,infile, files=None):
     """
     initialize an object
     """
     schema = None
     if files is None:
         files = Files()
     schema = files.get_schema("TESTS_SPEC_FILE")
     GenericXML.__init__(self, infile, schema=schema)

    def __init__(self,  infile=None, files=None):
        """
        initialize an object interface to file config_tests.xml
        """
        if infile is None:
            if files is None:
                files = Files()
            infile = files.get_value("CONFIG_TESTS_FILE")

        GenericXML.__init__(self,  infile)

 def __init__(self,infile, files=None):
     """
     initialize an object
     """
     schema = None
     if files is None:
         files = Files()
     schema = files.get_schema("TESTS_SPEC_FILE")
     GenericXML.__init__(self, infile, schema=schema)
     expect(self.get_version() >= 2.0,
            "{} is an unsupported version of the testfile format and will be ignored".format(infile))

    def _create_newcase_phase(self, test):
    ###########################################################################
        test_dir = self._get_test_dir(test)

        _, case_opts, grid, compset,\
            machine, compiler, test_mods = CIME.utils.parse_test_name(test)

        create_newcase_cmd = "%s --case %s --res %s --mach %s --compiler %s --compset %s"\
                               " --test" % \
                              (os.path.join(self._cime_root, "scripts", "create_newcase"),
                               test_dir, grid, machine, compiler, compset)
        if self._project is not None:
            create_newcase_cmd += " --project %s " % self._project
        if self._output_root is not None:
            create_newcase_cmd += " --output-root %s " % self._output_root


        if test_mods is not None:
            files = Files()
            (component,modspath) = test_mods.split('/',1)
            testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})

            test_mod_file = os.path.join(testmods_dir, component, modspath)
            if not os.path.exists(test_mod_file):
                self._log_output(test, "Missing testmod file '%s'" % test_mod_file)
                return False
            create_newcase_cmd += " --user-mods-dir %s" % test_mod_file

        if case_opts is not None:
            for case_opt in case_opts: # pylint: disable=not-an-iterable
                if case_opt.startswith('M'):
                    mpilib = case_opt[1:]
                    create_newcase_cmd += " --mpilib %s" % mpilib
                    logger.debug (" MPILIB set to %s" % mpilib)
                if case_opt.startswith('N'):
                    ninst = case_opt[1:]
                    create_newcase_cmd += " --ninst %s" %ninst
                    logger.debug (" NINST set to %s" % ninst)
                if case_opt.startswith('P'):
                    pesize = case_opt[1:]
                    create_newcase_cmd += " --pecount %s"%pesize

        if self._queue is not None:
            create_newcase_cmd += " --queue=%s" % self._queue

        if self._walltime is not None:
            create_newcase_cmd += " --walltime %s" % self._walltime
        elif test in self._test_data and "options" in self._test_data[test] and \
                "wallclock" in self._test_data[test]['options']:
            create_newcase_cmd += " --walltime %s" % self._test_data[test]['options']['wallclock']

        logger.debug("Calling create_newcase: " + create_newcase_cmd)
        return self._shell_cmd_for_phase(test, create_newcase_cmd, CREATE_NEWCASE_PHASE)