Python OperationService.prepare_operations_for_workflowsteps方法代码示例

# 需要导入模块: from tvb.core.services.operation_service import OperationService [as 别名]
# 或者: from tvb.core.services.operation_service.OperationService import prepare_operations_for_workflowsteps [as 别名]
class TestWorkflow(TransactionalTestCase):
    """
    Test that workflow conversion methods are valid.
    """


    def transactional_setup_method(self):
        """
        Sets up the testing environment;
        saves config file;
        creates a test user, a test project;
        creates burst, operation, flow and workflow services
        """
        self.test_user = TestFactory.create_user()
        self.test_project = TestFactory.create_project(self.test_user)
        self.workflow_service = WorkflowService()
        self.burst_service = BurstService()
        self.operation_service = OperationService()
        self.flow_service = FlowService()


    def transactional_teardown_method(self):
        """
        Remove project folders and clean up database.
        """
        FilesHelper().remove_project_structure(self.test_project.name)
        self.delete_project_folders()


    def __create_complex_workflow(self, workflow_step_list):
        """
        Creates a burst with a complex workflow with a given list of workflow steps.
        :param workflow_step_list: a list of workflow steps that will be used in the
            creation of a new workflow for a new burst
        """
        burst_config = TestFactory.store_burst(self.test_project.id)

        stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(Datatype1())

        first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class("tvb.tests.framework.adapters.testadapter1",
                                                                                   "TestAdapterDatatypeInput")
        metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
        kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
        operations, group = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id,
                                                                      first_step_algorithm,
                                                                      first_step_algorithm.algorithm_category,
                                                                      metadata, **kwargs)

        workflows = self.workflow_service.create_and_store_workflow(project_id=self.test_project.id,
                                                                    burst_id=burst_config.id,
                                                                    simulator_index=0,
                                                                    simulator_id=first_step_algorithm.id,
                                                                    operations=operations)
        self.operation_service.prepare_operations_for_workflowsteps(workflow_step_list, workflows, self.test_user.id,
                                                                    burst_config.id, self.test_project.id, group,
                                                                    operations)
        #fire the first op
        if len(operations) > 0:
            self.operation_service.launch_operation(operations[0].id, False)
        return burst_config.id


    def test_workflow_generation(self):
        """
        A simple test just for the fact that a workflow is created an ran, 
        no dynamic parameters are passed. In this case we create a two steps
        workflow: step1 - tvb.tests.framework.adapters.testadapter2.TestAdapter2
                  step2 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
        The first adapter doesn't return anything and the second returns one
        tvb.datatypes.datatype1.Datatype1 instance. We check that the steps
        are actually ran by checking that two operations are created and that
        one dataType is stored.
        """
        workflow_step_list = [TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2",
                                                               "TestAdapter2", step_index=1,
                                                               static_kwargs={"test2": 2}),
                              TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter1",
                                                               "TestAdapter1", step_index=2,
                                                               static_kwargs={"test1_val1": 1, "test1_val2": 1})]
        self.__create_complex_workflow(workflow_step_list)
        stored_datatypes = dao.get_datatypes_in_project(self.test_project.id)
        assert  len(stored_datatypes) == 2, "DataType from second step was not stored."
        assert  stored_datatypes[0].type == 'Datatype1', "Wrong type was stored."
        assert  stored_datatypes[1].type == 'Datatype1', "Wrong type was stored."

        finished, started, error, _, _ = dao.get_operation_numbers(self.test_project.id)
        assert  finished == 3, "Didnt start operations for both adapters in workflow."
        assert  started == 0, "Some operations from workflow didnt finish."
        assert  error == 0, "Some operations finished with error status."


    def test_workflow_dynamic_params(self):
        """
        A simple test just for the fact that dynamic parameters are passed properly
        between two workflow steps: 
                  step1 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
                  step2 - tvb.tests.framework.adapters.testadapter3.TestAdapter3
        The first adapter returns a tvb.datatypes.datatype1.Datatype1 instance. 
        The second adapter has this passed as a dynamic workflow parameter.
        We check that the steps are actually ran by checking that two operations 
#.........这里部分代码省略.........

# 需要导入模块: from tvb.core.services.operation_service import OperationService [as 别名]
# 或者: from tvb.core.services.operation_service.OperationService import prepare_operations_for_workflowsteps [as 别名]

#.........这里部分代码省略.........
    def _prepare_operations(self, burst_config, simulator_index, simulator_id, user_id):
        """
        Prepare all required operations for burst launch.
        """
        project_id = burst_config.fk_project
        burst_id = burst_config.id
        workflow_step_list = []
        starting_index = simulator_index + 1

        sim_algo = FlowService().get_algorithm_by_identifier(simulator_id)
        metadata = {DataTypeMetaData.KEY_BURST: burst_id}
        launch_data = burst_config.get_all_simulator_values()[0]
        operations, group = self.operation_service.prepare_operations(
            user_id, project_id, sim_algo, sim_algo.algo_group.group_category, metadata, **launch_data
        )
        group_launched = group is not None
        if group_launched:
            starting_index += 1

        for tab in burst_config.tabs:
            for portlet_cfg in tab.portlets:
                ### For each portlet configuration stored, update the step index ###
                ### and also change the dynamic parameters step indexes to point ###
                ### to the simulator outputs.                                     ##
                if portlet_cfg is not None:
                    analyzers = portlet_cfg.analyzers
                    visualizer = portlet_cfg.visualizer
                    for entry in analyzers:
                        entry.step_index = starting_index
                        self.workflow_service.set_dynamic_step_references(entry, simulator_index)
                        workflow_step_list.append(entry)
                        starting_index += 1
                    ### Change the dynamic parameters to point to the last adapter from this portlet execution.
                    visualizer.step_visible = False
                    if len(workflow_step_list) > 0 and isinstance(workflow_step_list[-1], model.WorkflowStep):
                        self.workflow_service.set_dynamic_step_references(visualizer, workflow_step_list[-1].step_index)
                    else:
                        self.workflow_service.set_dynamic_step_references(visualizer, simulator_index)
                    ### Only for a single operation have the step of visualization, otherwise is useless.
                    if not group_launched:
                        workflow_step_list.append(visualizer)

        if group_launched:
            ###  For a group of operations, make sure the metric for PSE view
            ### is also computed, immediately after the simulation.
            metric_algo, metric_group = FlowService().get_algorithm_by_module_and_class(
                MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS
            )
            _, metric_interface = FlowService().prepare_adapter(project_id, metric_group)
            dynamics = {}
            for entry in metric_interface:
                # We have a select that should be the dataType and a select multiple with the
                # required metric algorithms to be evaluated. Only dynamic parameter should be
                # the select type.
                if entry[ABCAdapter.KEY_TYPE] == "select":
                    dynamics[entry[ABCAdapter.KEY_NAME]] = {
                        WorkflowStepConfiguration.DATATYPE_INDEX_KEY: 0,
                        WorkflowStepConfiguration.STEP_INDEX_KEY: simulator_index,
                    }
            metric_step = model.WorkflowStep(
                algorithm_id=metric_algo.id, step_index=simulator_index + 1, static_param={}, dynamic_param=dynamics
            )
            metric_step.step_visible = False
            workflow_step_list.insert(0, metric_step)

        workflows = self.workflow_service.create_and_store_workflow(
            project_id, burst_id, simulator_index, simulator_id, operations
        )
        self.operation_service.prepare_operations_for_workflowsteps(
            workflow_step_list, workflows, user_id, burst_id, project_id, group, operations
        )
        operation_ids = [operation.id for operation in operations]
        return operation_ids

    def _async_launch_and_prepare(self, burst_config, simulator_index, simulator_id, user_id):
        """
        Prepare operations asynchronously.
        """
        try:
            operation_ids = self._prepare_operations(burst_config, simulator_index, simulator_id, user_id)
            self.logger.debug("Starting a total of %s workflows" % (len(operation_ids)))
            wf_errs = 0
            for operation_id in operation_ids:
                try:
                    OperationService().launch_operation(operation_id, True)
                except Exception, excep:
                    self.logger.error(excep)
                    wf_errs += 1
                    self.workflow_service.mark_burst_finished(burst_config, error_message=str(excep))

            self.logger.debug(
                "Finished launching workflows. "
                + str(len(operation_ids) - wf_errs)
                + " were launched successfully, "
                + str(wf_errs)
                + " had error on pre-launch steps"
            )
        except Exception, excep:
            self.logger.error(excep)
            self.workflow_service.mark_burst_finished(burst_config, error_message=str(excep))

# 需要导入模块: from tvb.core.services.operation_service import OperationService [as 别名]
# 或者: from tvb.core.services.operation_service.OperationService import prepare_operations_for_workflowsteps [as 别名]

#.........这里部分代码省略.........
                        self.workflow_service.set_dynamic_step_references(entry, simulator_index)
                        workflow_step_list.append(entry)
                        starting_index += 1
                    ### Change the dynamic parameters to point to the last adapter from this portlet execution.
                    visualizer.step_visible = False
                    if len(workflow_step_list) > 0 and isinstance(workflow_step_list[-1], model.WorkflowStep):
                        self.workflow_service.set_dynamic_step_references(visualizer, workflow_step_list[-1].step_index)
                    else:
                        self.workflow_service.set_dynamic_step_references(visualizer, simulator_index)
                    ### Only for a single operation have the step of visualization, otherwise is useless.
                    if not group_launched:
                        workflow_step_list.append(visualizer)

        if group_launched:
            ###  For a group of operations, make sure the metric for PSE view 
            ### is also computed, immediately after the simulation.
            metric_algo = FlowService().get_algorithm_by_module_and_class(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS)
            metric_interface = FlowService().prepare_adapter(project_id, metric_algo)
            dynamics = {}
            for entry in metric_interface:
                # We have a select that should be the dataType and a select multiple with the 
                # required metric algorithms to be evaluated. Only dynamic parameter should be
                # the select type.
                if entry[KEY_TYPE] == TYPE_SELECT:
                    dynamics[entry[KEY_NAME]] = {WorkflowStepConfiguration.DATATYPE_INDEX_KEY: 0,
                                                 WorkflowStepConfiguration.STEP_INDEX_KEY: simulator_index}
            metric_step = model.WorkflowStep(algorithm_id=metric_algo.id, step_index=simulator_index + 1,
                                             static_param={}, dynamic_param=dynamics)
            metric_step.step_visible = False
            workflow_step_list.insert(0, metric_step)

        workflows = self.workflow_service.create_and_store_workflow(project_id, burst_id, simulator_index,
                                                                    simulator_id, operations)
        self.operation_service.prepare_operations_for_workflowsteps(workflow_step_list, workflows, user_id,
                                                                    burst_id, project_id, group, operations)
        operation_ids = [operation.id for operation in operations]
        return operation_ids


    def _async_launch_and_prepare(self, burst_config, simulator_index, simulator_id, user_id):
        """
        Prepare operations asynchronously.
        """
        try:
            operation_ids = self._prepare_operations(burst_config, simulator_index, simulator_id, user_id)
            self.logger.debug("Starting a total of %s workflows" % (len(operation_ids, )))
            wf_errs = 0
            for operation_id in operation_ids:
                try:
                    OperationService().launch_operation(operation_id, True)
                except Exception as excep:
                    self.logger.error(excep)
                    wf_errs += 1
                    self.workflow_service.mark_burst_finished(burst_config, error_message=str(excep))

            self.logger.debug("Finished launching workflows. " + str(len(operation_ids) - wf_errs) +
                              " were launched successfully, " + str(wf_errs) + " had error on pre-launch steps")
        except Exception as excep:
            self.logger.error(excep)
            self.workflow_service.mark_burst_finished(burst_config, error_message=str(excep))


    @staticmethod
    def launch_visualization(visualization, frame_width=None, frame_height=None, is_preview=True):
        """
        :param visualization: a visualization workflow step