Python RunManager.setup_new_run方法代码示例

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    def test_run(self):
 
        # The paths work as follows: opus_matsim.__path__ is the path of the opus_matsim python module.  So we can use that
        # as anchor ...
        config_location = os.path.join(opus_matsim.__path__[0], 'tests')
        print "location: ", config_location
        run_config = XMLConfiguration( os.path.join(config_location,"test_config.xml")).get_run_configuration("Test")
        
        run_config['creating_baseyear_cache_configuration'].cache_directory_root = self.temp_dir
        run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = \
            os.path.join(opus_matsim.__path__[0], 'tests', 'testdata', 'base_year_data')

        # insert_auto_generated_cache_directory... does things I don't understand.  Need to do the following to obtain consistent
        # behavior independent from the file root:
        run_config['cache_directory'] = None
        
        insert_auto_generated_cache_directory_if_needed(run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())
    
        run_manager.setup_new_run(cache_directory = run_config['cache_directory'],
                                  configuration = run_config)
        
        run_manager.run_run(run_config, run_as_multiprocess = True )
        

        self.assert_(True)
        
        self.cleanup_test_run()

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
 def test_run(self):
     print "Entering test run"
     
     run_manager = RunManager(ServicesDatabaseConfiguration())
     run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
     
     run_manager.run_run(self.config, run_as_multiprocess = True )
     
     print "Leaving test run"

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    def run(self, config, executable):
        #--config=opus_matsim/sustain_city/configs/seattle_parcel.xml --executable=Seattle_baseline
        config = XMLConfiguration(config).get_run_configuration(executable)
        
        insert_auto_generated_cache_directory_if_needed(config)
     
        run_manager = RunManager(ServicesDatabaseConfiguration())
        
        run_manager.setup_new_run(cache_directory = config['cache_directory'],configuration = config)

        run_manager.run_run(config, run_as_multiprocess = True )

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    def run(self):
        
        logger.start_block()
        insert_auto_generated_cache_directory_if_needed(self.config)
         
        run_manager = RunManager(ServicesDatabaseConfiguration())
        run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
        
        run_manager.run_run(self.config, run_as_multiprocess = True )

        logger.end_block()

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
 def test_simulation(self):
     services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                  database_configuration = 'services_database_server' )
     run_manager = RunManager(services_db)
     run_as_multiprocess = True
     for scenario_name in ['san_antonio_baseline_test']:
         config = self.xml_config.get_run_configuration(scenario_name)
         insert_auto_generated_cache_directory_if_needed(config)
         run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                   configuration = config)
         run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
 def test_simulation(self):
     eugene_dir = __import__('eugene').__path__[0]
     xml_config = XMLConfiguration(os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Eugene_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
def prepare_run_manager(option_group=None):
    if option_group is None:
        option_group = StartRunOptionGroup()
    parser = option_group.parser
    options, args = option_group.parse()

    run_manager = RunManager(option_group.get_services_database_configuration(options))
    
    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
        insert_auto_generated_cache_directory_if_needed(config)
    elif options.xml_configuration is not None:
        if options.scenario_name is None:
            parser.print_help()
            sys.exit(1)
        config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name)
        insert_auto_generated_cache_directory_if_needed(config)
    else:
        parser.print_help()
        sys.exit(1)
        
    if options.existing_cache_to_copy is not None:
        config['creating_baseyear_cache_configuration'].cache_from_database = False
        config['creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
            existing_cache_to_copy = options.existing_cache_to_copy,
            )
        if options.years_to_cache is not None:
            config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache)

    if options.profile_filename is not None:
        config["profile_filename"] = options.profile_filename
 
    run_manager.setup_new_run(cache_directory = config['cache_directory'],
                              configuration = config)

    return options, config, run_manager

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
 def test_simulation(self):
     # check that the simulation proceeds without crashing
     # open the configuration for seattle_parcel.xml
     seattle_parcel_dir = __import__('seattle_parcel').__path__[0]
     xml_config = XMLConfiguration(os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Seattle_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    def test_simulation(self):
        base_year_data_path = os.path.join(self.data_path, 'base_year_data')        
        if not os.path.exists(base_year_data_path):
            os.makedirs(base_year_data_path)

        ftp_url = os.environ["FTP_URL"]
        file_name = os.path.split(ftp_url)[1]
        ftp_user = os.environ["FTP_USERNAME"]
        ftp_password = os.environ["FTP_PASSWORD"]
        
        #stdout, stderr = Popen("ls -la %s" % base_year_data_path, shell=True).communicate()
        #stdout, stderr = Popen("echo '%s'" % (base_year_data_path), stdout=PIPE).communicate()
        #print stdout
        
        try:
            Popen( """
                        cd %s;
                        pwd;
                        ls -la;
                        echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1;
                        rm -rf 2008;
                        unzip -o %s
                        """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name),
                        shell = True
                        ).communicate()
        except:
            print "Error when downloading and unzipping file from %s." % ftp_url
            raise

        services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                     database_configuration = 'services_database_server' )
        run_manager = RunManager(services_db)
        run_as_multiprocess = True
        xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'washtenaw_parcel.xml'))
        for scenario_name in ['washtenaw_baseline_test']:
            config = xml_config.get_run_configuration(scenario_name)
            insert_auto_generated_cache_directory_if_needed(config)
#            base_year = config['base_year']
#            config['years_to_run'] = (base_year+1, base_year+2)
            run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                      configuration = config)
            run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
    """Runs model system with a single model (for speed).
    Sets the .resources property of the caller before starting the run.
    """
    runs_manager = RunManager(config)

    run_configuration = SubsetConfiguration()
    run_configuration['creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
    run_configuration['models'] = ['land_price_model']
    if end_year is not None:
        run_configuration['years'] = (run_configuration['years'][0], end_year)
    
    SessionConfiguration(new_instance=True,
                         package_order=run_configuration['dataset_pool_configuration'].package_order,
                         in_storage=AttributeCache())
    insert_auto_generated_cache_directory_if_needed(run_configuration)
    caller.resources = run_configuration
    runs_manager.setup_new_run(cache_directory = run_configuration['cache_directory'],
                               configuration = run_configuration)
    runs_manager.run_run(run_configuration)

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    def testName(self):
        print "entering test_run"

        logger.log_status("Preparing MATsim test run ...")
        # unzip MATSim files
        matsim_zip = ExtractZipFile(self.matsim_source, self.destination)
        matsim_zip.extract()
        matsim_extracted_files = os.path.join(self.destination, "MATSimTestClasses")  # location of unziped MATSim files
        # unzip base_year_cache
        base_year_data_zip = ExtractZipFile(self.base_year_data_source, self.destination)
        base_year_data_zip.extract()
        base_year_data_extracted_files = os.path.join(
            self.destination, "base_year_data"
        )  # location of unziped base_year_cache

        # updating location of base_year_data
        self.run_config["creating_baseyear_cache_configuration"].cache_directory_root = self.destination
        self.run_config[
            "creating_baseyear_cache_configuration"
        ].baseyear_cache.existing_cache_to_copy = base_year_data_extracted_files
        self.run_config["cache_directory"] = base_year_data_extracted_files
        self.run_config.add("matsim_files", matsim_extracted_files)
        self.run_config.add("matsim_config", self.matsim_config_full)
        self.run_config.add("root", self.destination)

        insert_auto_generated_cache_directory_if_needed(self.run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())

        run_manager.setup_new_run(cache_directory=self.run_config["cache_directory"], configuration=self.run_config)

        logger.log_status("Strating UrbanSim run ... ")
        run_manager.run_run(self.run_config, run_as_multiprocess=True)
        # after the UrbanSim run the travel data sets schould be equal
        # self.assertTrue( self.compare_travel_data_sets() )
        logger.log_status("... UrbanSim run finished.")

        print "leaving test_run"

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]
    number_of_runs = config.get("number_of_runs", 1)
    number_of_runs_in_parallel = min(config.get("parallel_runs", 1), number_of_runs)
    # generate seeds for multiple runs
    root_seed = config.get("seed", None)
    seed(root_seed)
    # generate different seed for each run (each seed contains 1 number)
    seed_array = randint(1,2**30, number_of_runs)
    list_of_cache_directories = []
    for irun in range(number_of_runs):
        config['seed']= (seed_array[irun],)
        this_config = config.copy()
        if ((irun + 1) % number_of_runs_in_parallel) == 0:
            run_in_background = False
        else:
            run_in_background = True
        run_manager.setup_new_run(cache_directory = this_config['cache_directory'],
                                  configuration = this_config)
        run_manager.run_run(this_config, run_as_multiprocess=False,
                            run_in_background=run_in_background)
        if irun == 0:
            # log file for the multiple runs will be located in the first cache
            first_cache_directory = this_config['cache_directory']
            log_file = os.path.join(first_cache_directory, 'multiple_runs.log')
            logger.enable_file_logging(log_file)
            logger.log_status("Multiple runs: %s replications" % number_of_runs)
            logger.log_status("root random seed = %s" % str(root_seed))
        else:
            logger.enable_file_logging(log_file, verbose=False)

        logger.log_status("Run %s: %s" % (irun+1, this_config['cache_directory']))
        logger.disable_file_logging(log_file)
        list_of_cache_directories.append(this_config['cache_directory'])

# 需要导入模块: from opus_core.services.run_server.run_manager import RunManager [as 别名]
# 或者: from opus_core.services.run_server.run_manager.RunManager import setup_new_run [as 别名]

#.........这里部分代码省略.........
        else:
            raise ValueError, "Unrecognized optimizer {}".format(optimizer)

        default_kwargs.update(optimizer_kwargs)
        results = optimizer_func(self.target_func, copy(init_v), **default_kwargs)

        duration = time.time() - t0
        if results_pickle_prefix is not None:
            pickle_file = "{}_{}.pickle".format(results_pickle_prefix, optimizer)
            pickle_file = os.path.join(self.log_directory, pickle_file)
            pickle.dump(results, open(pickle_file, "wb"))

        if is_parallelizable == True:
            set_parallel(False)

        logger.log_status("init target_func: {}".format(self.target_func(init_v)))
        logger.log_status("end target_func: {}".format(results[:]))  # which one?
        logger.log_status("outputs from optimizer: {}".format(results))
        logger.log_status("Execution time: {}".format(duration))

    def init_run(self, create_baseyear_cache=True):
        """ init run, get run_id & cache_directory. """
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config, scenario_name=self.scenario)
        # run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources["cache_directory"]
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory()
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        # run_id = 275
        # cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory

    def update_parameters(
        self, est_v, cache_directory, simulation_state, dataset_pool, calib_datasets, *args, **kwargs
    ):
        i_est_v = 0
        current_year = simulation_state.get_current_time()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(cache_directory)

        for dataset_name, calib in calib_datasets.iteritems():
            dataset, calib_attr, index = calib
            if type(calib_attr) == str:
                dtype = dataset[calib_attr].dtype
                dataset[calib_attr][index] = (est_v[i_est_v : i_est_v + index.size]).astype(dtype)
                i_est_v += index.size
            elif type(calib_attr) in (list, tuple):
                for attr in calib_attr:
                    dtype = dataset[attr].dtype
                    dataset[attr][index] = (est_v[i_est_v : i_est_v + index.size]).astype(dtype)
                    i_est_v += index.size