Python process_unit.ProcessUnit类代码示例

    def test_change_mapping(self):
        """ Test using multiple input datasets, like if you were calculating a change. """

        with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
            fake_file_1 = '/a/fake/file_1956_red.nc'
            fake_file_2 = '/a/fake/file_1981_red.nc'

            mock_glob.return_value = [fake_file_1, fake_file_2]
            
            first_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
                                              set([Constraint('date', ['1956'])]))

            second_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
                                               set([Constraint('date', ['1981'])]))

            # Overwrite the valid combinations for these mock datasets.
            first_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
                                                                  Constraint('date', ['1956'])])])

            second_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
                                                                   Constraint('date', ['1981'])])])
            
            
            the_process_unit = ProcessUnit([first_pattern_ds, second_pattern_ds],
                                           "/a/final/output/file_%start_date%_%end_date%_%colour%.txt",
                                           'echo', map_dict={'start_date': ('date', 0),
                                                             'end_date': ('date', 1)})
        
            ds_result = the_process_unit.execute(simulate=True)
            
            outfiles = [file_thing for file_thing in ds_result.files]
            self.assertEqual(len(outfiles), 1)

            expected_string = self.script_header + "mkdir -p /a/final/output\necho /a/fake/file_1956_red.nc /a/fake/file_1981_red.nc /a/final/output/file_1956_1981_red.txt\n"     
            self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())

    def test_model_correllation_2(self):
        """ This test is to try combining Constraints from two different DataSets.

        This uses the new merge_output keyword option.

        """

        mock_file_1 = ["/red_echidna"]
        mock_file_2 = ["/blue_echidna"]

        input_pattern = "/%colour%_%animal%"

        # Create our mock DataSets.
        with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
            mock_glob.return_value = mock_file_1
            test_ds_1 = PatternDataSet(input_pattern)

        with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
            mock_glob.return_value = mock_file_2
            test_ds_2 = PatternDataSet(input_pattern)

        # A ProcessUnit which merges the Constraint on colour.
        the_process = ProcessUnit([test_ds_1, test_ds_2], "/tmp/%animal%_%colour%.file",
                                  "echo", merge_output=["colour"])

        output_ds = the_process.execute(simulate=False)

        outfiles = [metafile for metafile in output_ds.files]

        self.assertEqual(len(outfiles), 1)

        self.assertEqual(outfiles[0].full_path, "/tmp/echidna_red-blue.file")

    def test_simple_mapping(self):
        """ Test using constraints from the input in the output pattern. """

        with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:

            mock_file_1 = '/a/fake/ACCESS1-0_tas_netCDF'
            mock_file_2 = '/a/fake/MIROC_tas_netCDF'
            mock_file_3 = '/a/fake/ACCESS1-0_pr_netCDF'

            mock_glob.return_value = [mock_file_1, mock_file_2, mock_file_3]
        
            a_pattern_ds = PatternDataSet(self.input_pattern)

            the_process_unit = ProcessUnit([a_pattern_ds], self.output_pattern,
                                           'echo', map_dict={'obs_model': ('gcm_model', 0)})
            
            ds_result = the_process_unit.execute(simulate=True)
           
            outfiles = [file_thing for file_thing in ds_result.files]
            self.assertEqual(len(outfiles), 3)

            expected_string = (self.script_header + "mkdir -p /a/fake/output\necho /a/fake/MIROC_tas_netCDF /a/fake/output/MIROC_tas_netCDF\n" +
                               "echo /a/fake/ACCESS1-0_tas_netCDF /a/fake/output/ACCESS1-0_tas_netCDF\n" +
                               "echo /a/fake/ACCESS1-0_pr_netCDF /a/fake/output/ACCESS1-0_pr_netCDF\n")
            
            self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())

    def compute(self):

        in_dataset1 = self.getInputFromPort('in_dataset1')
        in_dataset2 = self.getInputFromPort('in_dataset2')

        self.positional_args = []
        self.keyword_args = {}

        new_constraints_for_output = set([Constraint('extra_info', ['fldcor']),
                                          Constraint('suffix', ['nc'])])

        merge_val =  self.getInputFromPort('merge_constraints')
        if merge_val:
            extra_merge = [cons_name.strip() for cons_name in merge_val.split(',')]
        else:
            extra_merge = []
        
        this_process = ProcessUnit([in_dataset1, in_dataset2],
                                   self.out_pattern,
                                   self.command,
                                   new_constraints_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args,
                                   cons_keywords=self.keyword_args,
                                   merge_output=extra_merge)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception as e:
            raise vistrails_module.ModuleError(self, repr(e))
            
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)

    def compute(self):

        in_dataset1 = self.getInputFromPort('in_dataset1')
        in_dataset2 = self.getInputFromPort('in_dataset2')
        operation = self.getInputFromPort('operation')

        self.positional_args = [(operation, 0, 'raw'), ]
        self.keyword_args = {}

        new_constraints_for_output = set([Constraint('extra_info', [operation]),
                                          Constraint('suffix', ['nc']),
                                          ])
        
        this_process = ProcessUnit([in_dataset1, in_dataset2],
                                   self.out_pattern,
                                   self.command,
                                   new_constraints_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args,
                                   cons_keywords=self.keyword_args,
                                   merge_output=['model', 'institute'])

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception as e:
            raise vistrails_module.ModuleError(self, repr(e))
            
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)

    def compute(self):

        # Required input
        in_dataset = self.getInputFromPort("in_dataset")
        year_start = self.getInputFromPort("start_year")
        year_end = self.getInputFromPort("end_year")
        
        new_cons = set([Constraint('year_start', [year_start]),
                        Constraint('year_end', [year_end]),
                        Constraint('suffix', ['nc']),
                        Constraint('grid', ['native'])])

        cons_for_output = new_cons

        # Execute the seas_vars process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   cons_for_output,
                                   positional_args=self.positional_args,
                                   cons_keywords=self.keyword_args,
                                   execution_options=self._execution_options)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception, e:
            raise vistrails_module.ModuleError(self, e.output)

    def compute(self):

        in_dataset = self.getInputFromPort('cod_dataset')

        command = "${CWSL_CTOOLS}/sdm/sdmrun.py"

        sdm_config = configuration.cwsl_ctools_path + "/sdm/default.cfg"
        positional_args = [("dxt-gridded", 0, "raw"),
                           ("-c " + sdm_config, 0, "raw")]
        
        # The data is written out to the default
        # location.
        output_pattern = os.path.join(configuration.user_basepath,
                                      FileCreator.default_pattern(in_dataset.constraints) + ".nc")
        this_process = ProcessUnit([in_dataset],
                                   output_pattern,
                                   command,
                                   in_dataset.constraints,
                                   execution_options=self._required_modules,
                                   positional_args=positional_args)

        this_process.execute(simulate=configuration.simulate_execution)
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)

    def compute(self):

        # Required input
        in_dataset = self.getInputFromPort("in_dataset")
        variable_name = self.getInputFromPort("variable_name")
        self.positional_args=[(variable_name,0,'raw'),('model',1)]

        new_cons = set([Constraint('variable', [variable_name]),
                        Constraint('suffix', ['png']),
                        ])

        # Optional added constraints.
        try:
            # Add extra constraints if necessary.
            added_constraints = self.getInputFromPort('added_constraints')
            cons_for_output = new_cons.intersection(added_constraints)
        except vistrails_module.ModuleError:
            cons_for_output = new_cons


        # Execute the seas_vars process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   cons_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception, e:
            raise vistrails_module.ModuleError(self, e.output)

    def compute(self):

        in_dataset = self.getInputFromPort('in_dataset')
        try:
            # Add extra constraints if necessary.
            added_constraints = self.getInputFromPort('added_constraints')
        except vistrails_module.ModuleError:
            added_constraints = None


        command = self.getInputFromPort("command")
        output_pattern = self.getInputFromPort("output_pattern")

        cons_for_output = set()
        if added_constraints:
            cons_for_output = set.union(cons_for_output,
                                        set(added_constraints))

        # Do the stuff.
        this_process = ProcessUnit([in_dataset],
                                   self.output_pattern,
                                   command,
                                   cons_for_output)

        this_process.execute(simulate=configuration.simulate_execution)
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)
        self.setResult('out_constraints', str(process_output.constraints))

        # Unload the modules at the end.
        self.module_loader.unload(self.required_modules)

    def test_mapping_after_passing(self):
        """ Mapping of constraints should work if applied to the output of an earlier ProcessUnit. """

        # First create a basic process unit.
        with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
            
            mock_glob.return_value = ['/a/fake/file/red_1986_2005_kangaroo.nc']

            first_patternds = PatternDataSet('/a/fake/file/%colour%_%start%_%end%_%animal%.nc')
            first_process = ProcessUnit([first_patternds], '/a/second/file_pattern/%colour%_%start%_%end%_%animal%.nc',
                                        'echo')
            result = first_process.execute(simulate=True)
            
            # Then take the output of that process and apply a mapping.
            second_process = ProcessUnit([result], '/a/final/pattern/%colour%_%hist_start%_%hist_end%_%animal%.txt',
                                         'echo', map_dict={'hist_start': ('start', 0),
                                                           'hist_end': ('end', 0)})

            final_out = second_process.execute(simulate=True)

            expected_string = (self.script_header + "mkdir -p /a/final/output\necho" +
                               "/a/second/file_pattern/red_1986_2005_kangaroo.nc " + 
                               "/a/final/pattern/red_1986_2005_kangaroo.nc")

            self.assertEqual(expected_string, second_process.scheduler.job.to_str())

    def compute(self):

        in_dataset = self.getInputFromPort('in_dataset')
        method = self.getInputFromPort('method')
        grid = self.getInputFromPort('grid')

        self.positional_args = [(method, 0, 'raw'), (grid, 1, 'raw'), ]
        self.keyword_args = {}
        
        grid = grid.split('/')[-1]
        if len(grid.split('.')) > 1:  # i.e. a weights file as opposed to pre-defined grid
            grid_constraint = method+'-'+grid.split('.')[0]
        else:
            grid_constraint = method+'-'+grid

        new_constraints_for_output = set([Constraint('grid_info', [grid_constraint]),
                                          Constraint('suffix', ['nc']),
                                          ])
        
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   new_constraints_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args,
                                   cons_keywords=self.keyword_args)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception as e:
            raise vistrails_module.ModuleError(self, repr(e))
            
        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)

    def compute(self):

        in_dataset = self.getInputFromPort('in_dataset')
        west_lon = self.getInputFromPort('west_lon')
        east_lon = self.getInputFromPort('east_lon')
        south_lat = self.getInputFromPort('south_lat')
        north_lat = self.getInputFromPort('north_lat')

        self.positional_args = [('west_lon', 0),
                                ('east_lon', 1),
                                ('south_lat', 2),
                                ('north_lat', 3)]
        self.keyword_args = {}

        new_constraints_for_output = set([Constraint('west_lon', [west_lon]),
                                          Constraint('east_lon', [east_lon]),
                                          Constraint('south_lat', [south_lat],
                                          Constraint('north_lat', [north_lat]))
                                          ])
        
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   new_constraints_for_output,
                                   execution_options=self._execution_options,
                                   positional_args=self.positional_args,
                                   cons_keywords=self.keyword_args)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except subprocess.CalledProcessError, e:
            raise vistrails_module.ModuleError(self, e.output)

    def compute(self):

        in_dataset = self.getInputFromPort("in_dataset")
        method = self.getInputFromPort("method")

        self.positional_args = [(method, 0, "raw")]
        self.keyword_args = {}

        if len(method.split(",")) > 1:
            agg_constraint = "".join(method.split(","))
        else:
            agg_constraint = method

        new_constraints_for_output = set([Constraint("latagg_info", [agg_constraint]), Constraint("suffix", ["nc"])])

        this_process = ProcessUnit(
            [in_dataset],
            self.out_pattern,
            self.command,
            new_constraints_for_output,
            execution_options=self._execution_options,
            positional_args=self.positional_args,
            cons_keywords=self.keyword_args,
        )

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except Exception as e:
            raise vistrails_module.ModuleError(self, repr(e))

        process_output = this_process.file_creator

        self.setResult("out_dataset", process_output)

    def compute(self):

        in_dataset = self.getInputFromPort('in_dataset')
        try:
            # Add extra constraints if necessary.
            added_constraints = self.getInputFromPort('added_constraints')
        except vistrails_module.ModuleError:
            added_constraints = None

        # Change the file_type constraint from nc to xml and add
        # any added constraints.
        cons_for_output = set([Constraint('suffix', ['xml'])])
        if added_constraints:
            cons_for_output = set.union(cons_for_output,
                                        set(added_constraints))


        # Execute the xml_to_nc process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   cons_for_output,
                                   execution_options=self._execution_options)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except subprocess.CalledProcessError, e:
            raise vistrails_module.ModuleError(self, e.output)

    def compute(self):

        # Required input
        in_dataset = self.getInputFromPort("in_dataset")

        new_cons = set([Constraint('extra_info', ['nino34']),
                        Constraint('latsouth_info', ['5S']),
                        Constraint('latnorth_info', ['5N']),
                        Constraint('latagg_info', ['fldavg']),
                        Constraint('lonwest_info', ['190E']),
                        Constraint('loneast_info', ['240E']),
                        Constraint('lonagg_info', ['fldavg']),
                        Constraint('leveltop_info', ['surface']),
                        Constraint('levelbottom_info', ['surface']),
                        Constraint('anomaly_info', ['anom']),
                       ])
        
        # Execute the process.
        this_process = ProcessUnit([in_dataset],
                                   self.out_pattern,
                                   self.command,
                                   new_cons,
                                   positional_args=self.positional_args,
                                   execution_options=self._execution_options)

        try:
            this_process.execute(simulate=configuration.simulate_execution)
        except subprocess.CalledProcessError as e:
            raise vistrails_module.ModuleError(self, e.output)
        except Exception as e:
            raise vistrails_module.ModuleError(self, repr(e))

        process_output = this_process.file_creator

        self.setResult('out_dataset', process_output)