Python base.Paths类代码示例

    def __init__(self, yaml_config_file):
        self.yaml_config_file = yaml_config_file
        self.root = Paths.dirname(self.yaml_config_file)
        self.yamls = self._get_all_yamls()
        self.cases = list()
        self.common_config = None

        # create dummy case for every yaml file in folder
        if not Paths.exists(self.yaml_config_file):
            self.common_config = deepcopy(DEFAULTS)
            for y in self.yamls:
                dummy_case = deepcopy(DEFAULTS)
                dummy_case['file'] = [y]
                self.cases.append(dummy_case)
        else:
            # setup common config values
            self.yaml_config = self._read_yaml()
            self.common_config = self.merge(DEFAULTS, self.yaml_config.get('common_config', {}))

            # first process files which are specified in test_cases
            missing = [Paths.basename(y) for y in self.yamls]
            for case in self.yaml_config.get('test_cases', []):
                case_config = self.merge(self.common_config, case)
                self.cases.append(case_config)
                for f in case_config['file']:
                    if f in missing:
                        missing.remove(f)

            # process rest (dummy case)
            for y in missing:
                dummy_case = deepcopy(self.common_config)
                dummy_case['file'] = [y]
                self.cases.append(dummy_case)

    def parse(self):
        for k, v in self.configs.items():
            self.configs[k] = ConfigBase(k)

        for k, v in self.files.items():
            config = Paths.join(Paths.dirname(k), yamlc.CONFIG_YAML)
            self.files[k] = self.configs[config]

def do_work(parser, args=None, debug=False):
    """
    :type args: list
    :type parser: utils.argparser.ArgParser
    """

    # parse arguments
    global arg_options, arg_others, arg_rest, debug_mode
    arg_options, arg_others, arg_rest = parser.parse(args)
    debug_mode = debug

    # configure path
    Paths.format = PathFormat.ABSOLUTE
    Paths.base_dir('' if not arg_options.root else arg_options.root)

    # check commands
    if len(arg_rest) == 0:
        parser.exit_usage('no MPI executable provided', exit_code=1)

    if len(arg_rest) == 1:
        parser.exit_usage('no executable provided', exit_code=2)

    # turn on dynamic messages if batch is not set
    Printer.dynamic_output = not arg_options.batch

    # # run local or pbs mode
    if arg_options.queue:
        return run_pbs_mode(debug)
    else:
        return run_local_mode(debug)

 def as_string(self):
     if self.file:
         return '{} x {}'.format(
             self.proc,
             Paths.path_end(Paths.without_ext(self.file), level=2)
         )
     return 'process'

 def get_command(f1, f2, **details):
     return [
         Paths.ndiff(),
         '-r', str(details.get('r_tol', '100.01')),
         '-a', str(details.get('a_tol', '100.0001')),
         Paths.abspath(f1),
         Paths.abspath(f2)
     ]

    def open(self):
        if self.mode in {self.SHOW, self.HIDE}:
            return {self.SHOW: None, self.HIDE: subprocess.PIPE}.get(self.mode)

        if self.mode in {self.WRITE, self.APPEND, self.VARIABLE}:
            if not self.fp:
                Paths.ensure_path(self.filename)
                self.fp = open(self.filename, 'w+' if self.mode is self.WRITE else 'a+')
            return self.fp

    def copy(self):
        # create dirs for target file
        Paths.ensure_path(self.target)

        # copy file
        shutil.copy(self.source, self.target)

        # remove file if set
        if self.remove_original:
            os.unlink(self.source)

 def get_command(f1, f2, **details):
     return [
         Paths.ndiff(),
         "-r",
         str(details.get("r_tol", "0.01")),
         "-a",
         str(details.get("a_tol", "0.0001")),
         Paths.abspath(f1),
         Paths.abspath(f2),
     ]

 def get_one(self, yaml_case_file):
     """
     :rtype: list[ConfigCase]
     """
     result = list()
     for case in self.cases:
         for f in case[yamlc.TAG_FILES]:
             if Paths.basename(f) == Paths.basename(yaml_case_file):
                 dummy_case = deepcopy(case)
                 dummy_case[yamlc.TAG_FILES] = [yaml_case_file]
                 result.extend(self._get_all_for_case(dummy_case))
     return [ConfigCase(r, self) for r in result]

    def _prepare(self):
        # configure printer
        Printer.batch_output = self.arg_options.batch
        Printer.dynamic_output = not self.arg_options.batch

        self.progress = Printer.dynamic_output
        self.batch = Printer.batch_output

        # configure path
        Paths.format = PathFormat.ABSOLUTE
        if self.arg_options.root:
            Paths.init(self.arg_options.root)

    def list_tests():
        test_dir = Paths.join(Paths.flow123d_root(), 'tests')
        tests = Paths.walk(test_dir, [
            PathFilters.filter_type_is_file(),
            PathFilters.filter_endswith('.yaml'),
            PathFilters.filter_not(PathFilters.filter_name('config.yaml')),
        ])
        result = dict()
        for r in tests:
            dirname = Paths.dirname(r)
            basename = Paths.basename(r)
            if Paths.dirname(dirname) != test_dir:
                continue

            if dirname not in result:
                result[dirname] = list()
            result[dirname].append(basename)
        keys = sorted(result.keys())

        for dirname in keys:
            Printer.all.out(Paths.relpath(dirname, test_dir))
            with Printer.all.with_level(1):
                for basename in result[dirname]:
                    Printer.all.out('{: >4s} {: <40s} {}', '', basename, Paths.relpath(Paths.join(dirname, basename), test_dir))
            Printer.all.newline()

    def open(self):
        if self.mode in {self.SHOW, self.HIDE}:
            return {self.SHOW: None, self.HIDE: subprocess.PIPE}.get(self.mode)

        if self.mode in {self.WRITE, self.APPEND, self.VARIABLE}:

            # open file manually when append or write
            if self.mode in {self.WRITE, self.APPEND}:
                Paths.ensure_path(self.filename)
                self.fp = open(self.filename, 'w+' if self.mode is self.WRITE else 'a+')

            # create temp file otherwise
            if self.mode is self.VARIABLE:
                self.fp, self.filename = tempfile.mkstemp()
            return self.fp

    def _get_ref_output_files(self, comp_data):
        """
        :type comp_data: dict
        """
        # parse filters
        filters = [PathFilters.filter_wildcards(x) for x in comp_data.get('files', [])]

        # browse files and make them relative to ref output so filters works properly
        files = Paths.walk(self.case.fs.ref_output, [PathFilters.filter_type_is_file()])
        files = [Paths.relpath(f, self.case.fs.ref_output) for f in files]

        # filter files and make them absolute again
        files = Paths.match(files, filters)
        files = [Paths.join(self.case.fs.ref_output, f) for f in files]
        return zip(files, self._get_mirror_files(files))

 def _get_flow123d(self):
     return [
         Paths.flow123d(),
         '-s', self.case.file,
         '-i', self.case.fs.input,
         '-o', self.case.fs.output
     ]

    def run_local_mode_one(self, proc):
        """
        Method runs single job with specified number of CPU
        :param proc:
        """
        if int(proc) == 0:
            command = self.rest[1:]
        else:
            command = [self.rest[0], '-np', proc] + self.rest[1:]

        n_lines = 0 if self.arg_options.batch else 10
        pypy = PyPy(BinExecutor(command))

        # set limits
        pypy.limit_monitor.time_limit = self.time_limit
        pypy.limit_monitor.memory_limit = self.memory_limit

        # catch output to variable
        # in batched mode we will keep the files
        # otherwise we will keep logs only on error
        log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log')
        pypy.executor.output = OutputMode.variable_output()
        pypy.full_output = log_file

        # save output to file
        pypy.output_monitor.log_file = log_file

        # start and wait for exit
        pypy.start()
        pypy.join()

        return pypy