Python FlowService.get_required_disk_size方法代码示例

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
    def test_launch_two_ops_HDD_full_space(self):
        """
        Launch two operations and give available space for user so that the first should finish,
        but after the update to the user hdd size the second should not.
        """
        module = "tvb.tests.framework.adapters.testadapter3"
        class_name = "TestAdapterHDDRequired"
        group = dao.find_group(module, class_name)
        adapter = FlowService().build_adapter_instance(group)
        output = adapter.get_output()
        output_type = output[0].__name__
        data = {"test": 100}
        TvbProfile.current.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data)))
        tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
                                                  tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
        dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
        self.assertEqual(len(dts), 1)
        datatype = dao.get_datatype_by_id(dts[0][0])
        self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
        self.assertEqual(datatype.type, output_type, "Wrong data stored.")
        #Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
        #plus what is estimated to be required from the next one (transform from B to MB)
        TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \
                                                float(adapter.get_required_disk_size(**data) - 1)

        self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
                          self.test_project.id, adapter,
                          tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
        dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
        self.assertEqual(len(dts), 1)

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
    def test_launch_two_ops_HDD_full_space(self):
        """
        Launch two operations and give available space for user so that the first should finish,
        but after the update to the user hdd size the second should not.
        """
        module = "tvb.tests.framework.adapters.testadapter3"
        class_name = "TestAdapterHDDRequired"
        group = dao.find_group(module, class_name)
        adapter = FlowService().build_adapter_instance(group)

        data = {"test": 100}
        TvbProfile.current.MAX_DISK_SPACE = 1 + float(adapter.get_required_disk_size(**data))
        tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)

        datatype = self._assert_stored_dt2()
        # Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
        # plus what is estimated to be required from the next one (transform from B to MB)
        TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + float(
            adapter.get_required_disk_size(**data) - 1
        )

        self.assertRaises(
            NoMemoryAvailableException,
            self.operation_service.initiate_operation,
            self.test_user,
            self.test_project.id,
            adapter,
            tmp_folder,
            **data
        )
        self._assert_stored_dt2()

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
 def test_launch_operation_HDD_full_space_started_ops(self):
     """
     Test the actual operation flow by executing a test adapter.
     """
     space_taken_by_started = 100
     module = "tvb.tests.framework.adapters.testadapter3"
     class_name = "TestAdapterHDDRequired"
     group = dao.find_group(module, class_name)
     started_operation = model.Operation(
         self.test_user.id,
         self.test_project.id,
         group.id,
         "",
         status=model.STATUS_STARTED,
         estimated_disk_size=space_taken_by_started,
     )
     dao.store_entity(started_operation)
     adapter = FlowService().build_adapter_instance(group)
     data = {"test": 100}
     TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) + space_taken_by_started - 1)
     tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
     self.assertRaises(
         NoMemoryAvailableException,
         self.operation_service.initiate_operation,
         self.test_user,
         self.test_project.id,
         adapter,
         tmp_folder,
         **data
     )
     self._assert_no_dt2()

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
    def test_launch_operation_HDD_with_space(self):
        """
        Test the actual operation flow by executing a test adapter.
        """
        module = "tvb.tests.framework.adapters.testadapter3"
        class_name = "TestAdapterHDDRequired"
        group = dao.find_group(module, class_name)
        adapter = FlowService().build_adapter_instance(group)
        data = {"test": 100}

        TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
        tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
        self._assert_stored_dt2()

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
    def test_launch_two_ops_HDD_with_space(self):
        """
        Launch two operations and give enough available space for user so that both should finish.
        """
        module = "tvb.tests.framework.adapters.testadapter3"
        class_name = "TestAdapterHDDRequired"
        group = dao.find_group(module, class_name)
        adapter = FlowService().build_adapter_instance(group)
        data = {"test": 100}
        TvbProfile.current.MAX_DISK_SPACE = 2 * float(adapter.get_required_disk_size(**data))
        tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")

        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
                                                  tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
        datatype = self._assert_stored_dt2()

        #Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB)
        #plus what is estimated to be required from the next one (transform from B to MB)
        TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float(adapter.get_required_disk_size(**data))

        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
                                                  tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
        self._assert_stored_dt2(2)

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
 def test_launch_operation_HDD_full_space(self):
     """
     Test the actual operation flow by executing a test adapter.
     """
     module = "tvb.tests.framework.adapters.testadapter3"
     class_name = "TestAdapterHDDRequired"
     group = dao.find_group(module, class_name)
     adapter = FlowService().build_adapter_instance(group)
     data = {"test": 100}
     TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) - 1)
     tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
     self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
                       self.test_project.id, adapter,
                       tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
     self._assert_no_dt2()

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
 def test_launch_operation_HDD_with_space(self):
     """
     Test the actual operation flow by executing a test adapter.
     """
     module = "tvb.tests.framework.adapters.testadapter3"
     class_name = "TestAdapterHDDRequired"
     group = dao.find_group(module, class_name)
     adapter = FlowService().build_adapter_instance(group)
     output = adapter.get_output()
     output_type = output[0].__name__
     data = {"test": 100}
     TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
     tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
     self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
                                               tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
     dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
     self.assertEqual(len(dts), 1)
     datatype = dao.get_datatype_by_id(dts[0][0])
     self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
     self.assertEqual(datatype.type, output_type, "Wrong data stored.")

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
    def test_delete_dt_free_HDD_space(self):
        """
        Launch two operations and give enough available space for user so that both should finish.
        """
        module = "tvb.tests.framework.adapters.testadapter3"
        class_name = "TestAdapterHDDRequired"
        group = dao.find_group(module, class_name)
        adapter = FlowService().build_adapter_instance(group)
        data = {"test": 100}
        TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
        tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")

        self._assert_no_dt2()
        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
        datatype = self._assert_stored_dt2()

        # Now free some space and relaunch
        ProjectService().remove_datatype(self.test_project.id, datatype.gid)
        self._assert_no_dt2()
        self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
        self._assert_stored_dt2()

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
class SimulatorAdapterTest(TransactionalTestCase):
    """
    Basic testing that Simulator is still working from UI.
    """
    CONNECTIVITY_NODES = 74

    def setUp(self):
        """
        Reset the database before each test.
        """
        initialize_storage()
        user = model.User("test_user", "test_pass", "[email protected]", True, "user")
        self.test_user = dao.store_entity(user)
        data = dict(name='test_proj', description='desc', users=[])
        self.test_project = ProjectService().store_project(self.test_user, True, None, **data)
        meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe",
                DataTypeMetaData.KEY_STATE: "INTERMEDIATE"}
        algo_group = dao.find_group(SIMULATOR_MODULE, SIMULATOR_CLASS)
        self.simulator_adapter = FlowService().build_adapter_instance(algo_group)

        self.operation = model.Operation(self.test_user.id, self.test_project.id, algo_group.id,
                                         json.dumps(SIMULATOR_PARAMETERS),
                                         meta=json.dumps(meta), status=model.STATUS_STARTED,
                                         method_name=ABCAdapter.LAUNCH_METHOD)
        self.operation = dao.store_entity(self.operation)

        SIMULATOR_PARAMETERS['connectivity'] = self._create_connectivity(self.CONNECTIVITY_NODES)


    def _create_connectivity(self, nodes_number):
        """
        Create a connectivity entity and return its GID
        """
        storage_path = FilesHelper().get_project_folder(self.test_project, str(self.operation.id))
        connectivity = Connectivity(storage_path=storage_path)
        connectivity.weights = numpy.ones((nodes_number, nodes_number))
        connectivity.centres = numpy.ones((nodes_number, 3))
        adapter_instance = StoreAdapter([connectivity])
        OperationService().initiate_prelaunch(self.operation, adapter_instance, {})

        return dao.get_datatype_by_id(connectivity.id).gid


    def test_happy_flow_launch(self):
        """
        Test that launching a simulation from UI works.
        """
        OperationService().initiate_prelaunch(self.operation, self.simulator_adapter, {}, **SIMULATOR_PARAMETERS)
        sim_result = dao.get_generic_entity(TimeSeriesRegion, 'TimeSeriesRegion', 'type')[0]
        self.assertEquals(sim_result.read_data_shape(), (32, 1, self.CONNECTIVITY_NODES, 1))


    def _estimate_hdd(self, new_parameters_dict):
        """ Private method, to return HDD estimation for a given set of input parameters"""
        filtered_params = self.simulator_adapter.prepare_ui_inputs(new_parameters_dict)
        self.simulator_adapter.configure(**filtered_params)
        return self.simulator_adapter.get_required_disk_size(**filtered_params)


    def test_estimate_hdd(self):
        """
        Test that occupied HDD estimation for simulation results considers simulation length.
        """
        factor = 5
        simulation_parameters = copy(SIMULATOR_PARAMETERS)
        ## Estimate HDD with default simulation parameters
        estimate1 = self._estimate_hdd(simulation_parameters)
        self.assertTrue(estimate1 > 1)

        ## Change simulation length and monitor period, we expect a direct proportial increase in estimated HDD
        simulation_parameters['simulation_length'] = float(simulation_parameters['simulation_length']) * factor
        period = float(simulation_parameters['monitors_parameters_option_TemporalAverage_period'])
        simulation_parameters['monitors_parameters_option_TemporalAverage_period'] = period / factor
        estimate2 = self._estimate_hdd(simulation_parameters)
        self.assertEqual(estimate1, estimate2 / factor / factor)

        ## Change number of nodes in connectivity. Expect HDD estimation increase.
        simulation_parameters['connectivity'] = self._create_connectivity(self.CONNECTIVITY_NODES * factor)
        estimate3 = self._estimate_hdd(simulation_parameters)
        self.assertEqual(estimate2, estimate3 / factor)


    def test_estimate_execution_time(self):
        """
        Test that get_execution_time_approximation considers the correct params
        """
        ## Compute reference estimation
        simulation_parameters = self.simulator_adapter.prepare_ui_inputs(SIMULATOR_PARAMETERS)
        estimation1 = self.simulator_adapter.get_execution_time_approximation(**simulation_parameters)

        ## Estimation when the surface input parameter is set
        simulation_parameters['surface'] = "GID_surface"
        estimation2 = self.simulator_adapter.get_execution_time_approximation(**simulation_parameters)

        self.assertEqual(estimation1, estimation2 / 500)
        simulation_parameters['surface'] = ""

        ## Modify integration step and simulation length:
        initial_simulation_length = float(simulation_parameters['simulation_length'])
        initial_integration_step = float(simulation_parameters['integrator_parameters']['dt'])
#.........这里部分代码省略.........

# 需要导入模块: from tvb.core.services.flow_service import FlowService [as 别名]
# 或者: from tvb.core.services.flow_service.FlowService import get_required_disk_size [as 别名]
class SimulatorAdapterTest(TransactionalTestCase):
    """
    Basic testing that Simulator is still working from UI.
    """
    CONNECTIVITY_NODES = 74

    def setUp(self):
        """
        Reset the database before each test.
        """
        initialize_storage()
        self.datatypes_factory = DatatypesFactory()
        self.test_user = self.datatypes_factory.get_user()
        self.test_project = self.datatypes_factory.get_project()
        self.connectivity = self.datatypes_factory.create_connectivity(self.CONNECTIVITY_NODES)[1]

        algo_group = dao.find_group(SIMULATOR_MODULE, SIMULATOR_CLASS)
        algorithm = dao.get_algorithm_by_group(algo_group.id)
        self.simulator_adapter = FlowService().build_adapter_instance(algo_group)
        self.operation = TestFactory.create_operation(algorithm, self.test_user, self.test_project,
                                                      model.STATUS_STARTED, json.dumps(SIMULATOR_PARAMETERS))

        SIMULATOR_PARAMETERS['connectivity'] = self.connectivity.gid


    def test_happy_flow_launch(self):
        """
        Test that launching a simulation from UI works.
        """
        OperationService().initiate_prelaunch(self.operation, self.simulator_adapter, {}, **SIMULATOR_PARAMETERS)
        sim_result = dao.get_generic_entity(TimeSeriesRegion, 'TimeSeriesRegion', 'type')[0]
        self.assertEquals(sim_result.read_data_shape(), (32, 1, self.CONNECTIVITY_NODES, 1))


    def _estimate_hdd(self, new_parameters_dict):
        """ Private method, to return HDD estimation for a given set of input parameters"""
        filtered_params = self.simulator_adapter.prepare_ui_inputs(new_parameters_dict)
        self.simulator_adapter.configure(**filtered_params)
        return self.simulator_adapter.get_required_disk_size(**filtered_params)


    def test_estimate_hdd(self):
        """
        Test that occupied HDD estimation for simulation results considers simulation length.
        """
        factor = 5
        simulation_parameters = copy(SIMULATOR_PARAMETERS)
        ## Estimate HDD with default simulation parameters
        estimate1 = self._estimate_hdd(simulation_parameters)
        self.assertTrue(estimate1 > 1)

        ## Change simulation length and monitor period, we expect a direct proportial increase in estimated HDD
        simulation_parameters['simulation_length'] = float(simulation_parameters['simulation_length']) * factor
        period = float(simulation_parameters['monitors_parameters_option_TemporalAverage_period'])
        simulation_parameters['monitors_parameters_option_TemporalAverage_period'] = period / factor
        estimate2 = self._estimate_hdd(simulation_parameters)
        self.assertEqual(estimate1, estimate2 / factor / factor)

        ## Change number of nodes in connectivity. Expect HDD estimation increase.
        large_conn_gid = self.datatypes_factory.create_connectivity(self.CONNECTIVITY_NODES * factor)[1].gid
        simulation_parameters['connectivity'] = large_conn_gid
        estimate3 = self._estimate_hdd(simulation_parameters)
        self.assertEqual(estimate2, estimate3 / factor)


    def test_estimate_execution_time(self):
        """
        Test that get_execution_time_approximation considers the correct params
        """
        ## Compute reference estimation
        params = self.simulator_adapter.prepare_ui_inputs(SIMULATOR_PARAMETERS)
        estimation1 = self.simulator_adapter.get_execution_time_approximation(**params)

        ## Estimation when the surface input parameter is set
        params['surface'] = "GID_surface"
        estimation2 = self.simulator_adapter.get_execution_time_approximation(**params)

        self.assertEqual(estimation1, estimation2 / 500)
        params['surface'] = ""

        ## Modify integration step and simulation length:
        initial_simulation_length = float(params['simulation_length'])
        initial_integration_step = float(params['integrator_parameters']['dt'])

        for factor in (2, 4, 10):
            params['simulation_length'] = initial_simulation_length * factor
            params['integrator_parameters']['dt'] = initial_integration_step / factor

            estimation3 = self.simulator_adapter.get_execution_time_approximation(**params)

            self.assertEqual(estimation1, estimation3 / factor / factor)

        ## Check that no division by zero happens
        params['integrator_parameters']['dt'] = 0
        estimation4 = self.simulator_adapter.get_execution_time_approximation(**params)
        self.assertTrue(estimation4 > 0)

        ## even with length zero, still a positive estimation should be returned
        params['simulation_length'] = 0
        estimation5 = self.simulator_adapter.get_execution_time_approximation(**params)
#.........这里部分代码省略.........