Python run_manager.RunManager类代码示例

def main(option_group=None, args=None):
    if option_group is None:
        option_group = RestartRunOptionGroup()
    parser = option_group.parser
    if args is None:
        options, args = option_group.parse()
    else: 
        options, _args = option_group.parse()
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process
    
    if len(args) < 2:
        parser.print_help()
    else:
        run_id, year = (int(args[0]), int(args[1]))
        end_year = int(options.end_year) if options.end_year is not None else None
        run_manager.restart_run(run_id, 
                                year,
                                options.project_name,
                                end_year=end_year,
                                skip_urbansim=options.skip_urbansim,
                                create_baseyear_cache_if_not_exists=options.create_baseyear_cache_if_not_exists,
                                skip_cache_cleanup=options.skip_cache_cleanup,
                                run_as_multiprocess=run_as_multiprocess
                                )

    def test_run(self):
 
        # The paths work as follows: opus_matsim.__path__ is the path of the opus_matsim python module.  So we can use that
        # as anchor ...
        config_location = os.path.join(opus_matsim.__path__[0], 'tests')
        print "location: ", config_location
        run_config = XMLConfiguration( os.path.join(config_location,"test_config.xml")).get_run_configuration("Test")
        
        run_config['creating_baseyear_cache_configuration'].cache_directory_root = self.temp_dir
        run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = \
            os.path.join(opus_matsim.__path__[0], 'tests', 'testdata', 'base_year_data')

        # insert_auto_generated_cache_directory... does things I don't understand.  Need to do the following to obtain consistent
        # behavior independent from the file root:
        run_config['cache_directory'] = None
        
        insert_auto_generated_cache_directory_if_needed(run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())
    
        run_manager.setup_new_run(cache_directory = run_config['cache_directory'],
                                  configuration = run_config)
        
        run_manager.run_run(run_config, run_as_multiprocess = True )
        

        self.assert_(True)
        
        self.cleanup_test_run()

 def test_run(self):
     print "Entering test run"
     
     run_manager = RunManager(ServicesDatabaseConfiguration())
     run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
     
     run_manager.run_run(self.config, run_as_multiprocess = True )
     
     print "Leaving test run"

 def test_simulation(self):
     services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                  database_configuration = 'services_database_server' )
     run_manager = RunManager(services_db)
     run_as_multiprocess = True
     for scenario_name in ['san_antonio_baseline_test']:
         config = self.xml_config.get_run_configuration(scenario_name)
         insert_auto_generated_cache_directory_if_needed(config)
         run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                   configuration = config)
         run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)

    def run(self, config, executable):
        #--config=opus_matsim/sustain_city/configs/seattle_parcel.xml --executable=Seattle_baseline
        config = XMLConfiguration(config).get_run_configuration(executable)
        
        insert_auto_generated_cache_directory_if_needed(config)
     
        run_manager = RunManager(ServicesDatabaseConfiguration())
        
        run_manager.setup_new_run(cache_directory = config['cache_directory'],configuration = config)

        run_manager.run_run(config, run_as_multiprocess = True )

    def run(self):
        
        logger.start_block()
        insert_auto_generated_cache_directory_if_needed(self.config)
         
        run_manager = RunManager(ServicesDatabaseConfiguration())
        run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
        
        run_manager.run_run(self.config, run_as_multiprocess = True )

        logger.end_block()

    def on_buttonBox_accepted(self):
        path = str(self.lePath.text())
        if not os.path.exists(path):
            msg = 'Cannot import, %s does not exist' % path
            logger.log_warning(msg)
            MessageBox.warning(mainwindow = self,
                            text = msg,
                            detailed_text = '')
        else:
            cache_directory = path
            years = []

            for dir in os.listdir(cache_directory):
                if len(dir) == 4 and dir.isdigit():
                    years.append(int(dir))
            if years == []:
                msg = 'Cannot import, %s has no run data'%path
                logger.log_warning(msg)
                MessageBox.warning(mainwindow = self,
                                text = msg,
                                detailed_text = '')

            else:
                start_year = min(years)
                end_year = max(years)
                project_name = os.environ['OPUSPROJECTNAME']
                run_name = os.path.basename(path)

                server_config = ServicesDatabaseConfiguration()
                run_manager = RunManager(server_config)

                run_id = run_manager._get_new_run_id()
                resources = {
                     'cache_directory': cache_directory,
                     'description': '',
                     'years': (start_year, end_year),
                     'project_name': project_name
                }

                try:
                    run_manager.add_row_to_history(run_id = run_id,
                                                   resources = resources,
                                                   status = 'done',
                                                   run_name = run_name)
                    update_available_runs(self.project)
                    logger.log_status('Added run %s of project %s to run_activity table'%(run_name, project_name))
                except:
                    errorInfo = formatExceptionInfo()
                    logger.log_error(errorInfo)
                    MessageBox.error(mainwindow = self,
                                    text = 'Could not add run %s of project %s to run_activity table'%(run_name, project_name),
                                    detailed_text = errorInfo)
        self.close()

 def test_simulation(self):
     eugene_dir = __import__('eugene').__path__[0]
     xml_config = XMLConfiguration(os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Eugene_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)

def prepare_run_manager(option_group=None):
    if option_group is None:
        option_group = StartRunOptionGroup()
    parser = option_group.parser
    options, args = option_group.parse()

    run_manager = RunManager(option_group.get_services_database_configuration(options))
    
    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
        insert_auto_generated_cache_directory_if_needed(config)
    elif options.xml_configuration is not None:
        if options.scenario_name is None:
            parser.print_help()
            sys.exit(1)
        config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name)
        insert_auto_generated_cache_directory_if_needed(config)
    else:
        parser.print_help()
        sys.exit(1)
        
    if options.existing_cache_to_copy is not None:
        config['creating_baseyear_cache_configuration'].cache_from_database = False
        config['creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
            existing_cache_to_copy = options.existing_cache_to_copy,
            )
        if options.years_to_cache is not None:
            config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache)

    if options.profile_filename is not None:
        config["profile_filename"] = options.profile_filename
 
    run_manager.setup_new_run(cache_directory = config['cache_directory'],
                              configuration = config)

    return options, config, run_manager

 def test_simulation(self):
     # check that the simulation proceeds without crashing
     # open the configuration for seattle_parcel.xml
     seattle_parcel_dir = __import__('seattle_parcel').__path__[0]
     xml_config = XMLConfiguration(os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Seattle_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)

    def init_run(self, create_baseyear_cache=True):
        """ init run, get run_id & cache_directory. """
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config, scenario_name=self.scenario)
        # run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources["cache_directory"]
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory()
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        # run_id = 275
        # cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory

def main(option_group=None, args=None):

    if option_group is None:
        option_group = restart_run.RestartRunOptionGroup()
    parser = option_group.parser
    if args is None:
        options, args = option_group.parse()
    else: 
        options, _args = option_group.parse()
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process

    if len(args) < 1:
        parser.print_help()
        sys.exit(1)
    run_id = int(args[0])
    year = None
    if len(args) > 1:
        year = int(args[1])

    # Get configuration from DB and ensure the run directory is mounted.  Note
    # that we pass a dummy value for restart_year because we might not know it
    # yet.  But when we actually restart the run, we will pass the correct
    # year.
    run_resources = run_manager.create_run_resources_from_history(run_id=run_id,
                                                                  restart_year=2010)
    hudson_common.mount_cache_dir(run_resources)

    cache_dir = run_resources['cache_directory']
    if not year:
        # guess the year based on how the cache dir is populated
        years = map(lambda y : int(os.path.basename(y)),
                    glob.glob(os.path.join(cache_dir, "2*")))
        year = max(years)

    end_year = int(options.end_year) if options.end_year is not None else None
    run_manager.restart_run(run_id, 
                            year,
                            options.project_name,
                            end_year=end_year,
                            skip_urbansim=options.skip_urbansim,
                            create_baseyear_cache_if_not_exists=options.create_baseyear_cache_if_not_exists,
                            skip_cache_cleanup=options.skip_cache_cleanup,
                            run_as_multiprocess=run_as_multiprocess
                            )

    def test_restart_simple_run(self):
        _do_run_simple_test_run(self, self.temp_dir, self.config, end_year=1983)
        runs_manager = RunManager(self.config)
        config = _get_run_config(temp_dir=self.temp_dir)
        runs_manager.update_environment_variables(run_resources=config)

        run_activity = runs_manager.services_db.get_table("run_activity")
        s = select([func.max(run_activity.c.run_id)])
        run_id = runs_manager.services_db.execute(s).fetchone()[0]

        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        runs_manager.restart_run(run_id, restart_year=1981, project_name="eugene_gridcell", skip_urbansim=False)

        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        # Restaring without running urbansim should not re-run that year.
        # TODO: test that no models are run this time.
        runs_manager.restart_run(run_id, restart_year=1982, project_name="eugene_gridcell", skip_urbansim=True)
        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        self.cleanup_test_run()

    def test_simulation(self):
        base_year_data_path = os.path.join(self.data_path, 'base_year_data')        
        if not os.path.exists(base_year_data_path):
            os.makedirs(base_year_data_path)

        ftp_url = os.environ["FTP_URL"]
        file_name = os.path.split(ftp_url)[1]
        ftp_user = os.environ["FTP_USERNAME"]
        ftp_password = os.environ["FTP_PASSWORD"]
        
        #stdout, stderr = Popen("ls -la %s" % base_year_data_path, shell=True).communicate()
        #stdout, stderr = Popen("echo '%s'" % (base_year_data_path), stdout=PIPE).communicate()
        #print stdout
        
        try:
            Popen( """
                        cd %s;
                        pwd;
                        ls -la;
                        echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1;
                        rm -rf 2008;
                        unzip -o %s
                        """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name),
                        shell = True
                        ).communicate()
        except:
            print "Error when downloading and unzipping file from %s." % ftp_url
            raise

        services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                     database_configuration = 'services_database_server' )
        run_manager = RunManager(services_db)
        run_as_multiprocess = True
        xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'washtenaw_parcel.xml'))
        for scenario_name in ['washtenaw_baseline_test']:
            config = xml_config.get_run_configuration(scenario_name)
            insert_auto_generated_cache_directory_if_needed(config)
#            base_year = config['base_year']
#            config['years_to_run'] = (base_year+1, base_year+2)
            run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                      configuration = config)
            run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)

def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
    """Runs model system with a single model (for speed).
    Sets the .resources property of the caller before starting the run.
    """
    runs_manager = RunManager(config)

    run_configuration = SubsetConfiguration()
    run_configuration['creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
    run_configuration['models'] = ['land_price_model']
    if end_year is not None:
        run_configuration['years'] = (run_configuration['years'][0], end_year)
    
    SessionConfiguration(new_instance=True,
                         package_order=run_configuration['dataset_pool_configuration'].package_order,
                         in_storage=AttributeCache())
    insert_auto_generated_cache_directory_if_needed(run_configuration)
    caller.resources = run_configuration
    runs_manager.setup_new_run(cache_directory = run_configuration['cache_directory'],
                               configuration = run_configuration)
    runs_manager.run_run(run_configuration)