Python AttributeCache.get_storage_location方法代码示例

# 需要导入模块: from opus_core.store.attribute_cache import AttributeCache [as 别名]
# 或者: from opus_core.store.attribute_cache.AttributeCache import get_storage_location [as 别名]
class DatasetSizeModel(Model):
    """Checks if all datasets after collapsing over all years have attributes of the same size."""
    
    def __init__(self, directory=None):
        if directory is None:
            directory = SimulationState().get_cache_directory()
        self.cache = AttributeCache(directory)
    
    def run(self):
        year_orig = SimulationState().get_current_time()
        years = self.years_in_cache()
        SimulationState().set_current_time(years[0])
        storages = {}
        for year in years:
            storages[year] = flt_storage(os.path.join(self.cache.get_storage_location(), '%s' % year))
        tables = self.cache.get_table_names()
        counts = pd.Series(np.zeros(len(tables), dtype="int32"), index=tables)
        for table in tables:
            columns = self.cache._get_column_names_and_years(table)
            values = []
            names = []
            colyears = []
            for col, year in columns:
                if col in names:
                    continue
                data = storages[year].load_table(table, column_names=col)
                values.append(data[col].size)
                names.append(col)
                colyears.append(year)
            values = np.array(values)
            if(all(values == values[0])):
                continue # all attributes have the same size
            # there is an inconsistency in attributes length
            names = np.array(names)
            colyears = np.array(colyears)
            uc = np.unique(values, return_counts=True)
            imax = np.argmax(uc[1])
            idx = np.where(values <> uc[0][imax])[0]
            df = pd.DataFrame({"column": names[idx],  "year": colyears[idx], "size": values[idx]})
            df = df.append(pd.DataFrame({"column": np.array(["all other columns"]), "year": np.array([years[0]]), "size": np.array([uc[0][imax]])}))
            logger.log_status("Inconsistency in table ", table, ":\n", df)
            counts[table] = df.shape[0] - 1
        SimulationState().set_current_time(year_orig)
        logger.log_status("Model total:", counts.sum(), ' size inconsistencies found.')
        return counts

    
    def years_in_cache(self):
        return self.cache._get_sorted_list_of_years(start_with_current_year=False)

# 需要导入模块: from opus_core.store.attribute_cache import AttributeCache [as 别名]
# 或者: from opus_core.store.attribute_cache.AttributeCache import get_storage_location [as 别名]
def import_travel_model_data(config, year):

    cache_directory = config['cache_directory']
    simulation_state = SimulationState()
    simulation_state.set_current_time(year)
    simulation_state.set_cache_directory(cache_directory)
    out_store = AttributeCache().get_flt_storage_for_year(year+1)
    out_store_loc = out_store.get_storage_location()

    tm_config = config['travel_model_configuration']
    data_to_import = tm_config['tm_to_urbansim_variable_mapping'] 
    base_dir = mtc_common.tm_get_base_dir(config)
    data_dir = tm_config[year]['data_dir']

    for dataset_name, skim_file in data_to_import.iteritems():
        skim_file = os.path.join(base_dir, data_dir, skim_file)
        data = read_csv(skim_file, header=0)
        
        with block("Caching {} to {}".format(dataset_name, out_store_loc)):
            logger.log_status("Source file {}".format(skim_file))
            opus_ds = to_opus_dataset(data, out_store, dataset_name)

# 需要导入模块: from opus_core.store.attribute_cache import AttributeCache [as 别名]
# 或者: from opus_core.store.attribute_cache.AttributeCache import get_storage_location [as 别名]
def import_openamos_data(config, year, zone_set=None):
    tm_config = config['travel_model_configuration']
    if tm_config.has_key('skim_dir'):
        skim_dir = tm_config.get('skim_dir')
    else:
        projectLoc = tm_config.get("project_path")

        #openamos_dir = tm_config[year]
        #skim_dir = "/workspace/workdata/SimTRAVEL_data/base_scenario/skims/bootstrap/"
        print "--->", projectLoc
        skim_dir = os.path.join(projectLoc, "skimOutput/dynamic")
    logger.log_status('Reading skims from {}'.format(skim_dir))
    skim_files = glob.glob(os.path.join(skim_dir, "skim*.dat"))
    print skim_files
    skims = None
    """
    for skim_file in skim_files:
        i = int( re.search('\d+', skim_file).group(0) )
        skim = read_csv_with_numpy(skim_file, header=False,
                                   columns=['from_zone_id', 'to_zone_id', str(i)])
        if skims is None:
            skims = skim
        else:
            import pdb; pdb.set_trace()
            skims = np.hstack((skims, skim[str(i)]))
    """
    attr_pattern = '{}{}'
    for skim_file in skim_files:
        i = int( re.findall('\d+', skim_file)[-1] )
        skim = read_csv(skim_file, header=0,
                        names=['from_zone_id', 'to_zone_id', 'travel_time', 'travel_distance'])
        if skims is None:
            skims = skim.rename(columns={'travel_time': attr_pattern.format('tm', str(i)),
                                         'travel_distance': attr_pattern.format('td', str(i)),
                                        }, copy=False)
        else:
            #skims.insert(i, str(i), skim.travel_time
            skims[attr_pattern.format('tm', str(i))] = skim.travel_time
            skims[attr_pattern.format('td', str(i))] = skim.travel_distance

    skims.set_index(['from_zone_id', 'to_zone_id'], inplace=True)

    peak_hours = set([6, 7, 8, 9, 16, 17, 18, 19])
    off_peak_hours = set(range(24)) - peak_hours
    peak_travel_time = avg_travel_time(skims, peak_hours, prefix='tm')
    off_peak_travel_time = avg_travel_time(skims, off_peak_hours, prefix='tm')
    peak_travel_distance = avg_travel_time(skims, peak_hours, prefix='td')
    off_peak_travel_distance = avg_travel_time(skims, off_peak_hours, prefix='td')
    travel_time = DataFrame({'peak_travel_time': peak_travel_time, 
                             'off_peak_travel_time': off_peak_travel_time,
                             'peak_travel_distance': peak_travel_distance, 
                             'off_peak_travel_distance': off_peak_travel_distance,
                            })

    ## subset to include only zones appearing in zone_set
    #zone_ids = zone_set['zone_id']
    #zone_pairs = [z for z in product(zone_ids, zone_ids)]
    #travel_time = travel_time.ix[zone_pairs]

    cache_directory = config['cache_directory']
    simulation_state = SimulationState()
    simulation_state.set_current_time(year)
    simulation_state.set_cache_directory(cache_directory)
    out_store = AttributeCache().get_flt_storage_for_year(year+1)
    logger.log_status('Caching travel_data to {}'.format(out_store.get_storage_location()))
    travel_data = to_opus_dataset(travel_time, out_store, 'travel_data')

    return travel_data

# 需要导入模块: from opus_core.store.attribute_cache import AttributeCache [as 别名]
# 或者: from opus_core.store.attribute_cache.AttributeCache import get_storage_location [as 别名]
class DataStructureModel(Model):
    """
    Checks the structure of datasets in a given cache (or run cache) when compared to a reference cache.
    It writes out all columns that are missing as well as those that are not present in the reference cache.
    It can also compare the sizes of the datasets. 
    """
    def __init__(self, reference_location=None):
        """
        "reference_location" is the directory of the reference cache and should include the year.
        If it is None, the simulation directory in its start year is taken. 
        """
        if reference_location is None:
            reference_location = os.path.join(SimulationState().get_cache_directory(), "%s" % SimulationState().get_start_time())
        self.reference_storage =  flt_storage(reference_location)
    
    def run(self, directory=None, check_size=True):
        """
        "directory" is the cache to be compared to the reference. It should not include the year
        as the model checks all years.
        Set "check_sizes" to False if no size check of the datasets is required. 
        """
        if directory is None:
            directory = SimulationState().get_cache_directory()        
        self.cache = AttributeCache(directory)
        year_orig = SimulationState().get_current_time()
        years = self.years_in_cache()
        SimulationState().set_current_time(years[0])
        storages = {}
        for year in years:
            storages[year] = flt_storage(os.path.join(self.cache.get_storage_location(), '%s' % year))
        df = pd.DataFrame(columns=["Table", "Less-than-ref", "More-than-ref", "Year", "Size", "Size-ref"])
        tables = self.cache.get_table_names() 
        for table in tables:
            columns_list = self.cache.get_column_names(table)
            columns = Set(columns_list)
            ref_columns_list = self.reference_storage.get_column_names(table, lowercase=True)
            ref_columns = Set(ref_columns_list)
            more = columns.difference(ref_columns)
            less = ref_columns.difference(columns)
            samesize = True
            if check_size:
                table_size = self.cache.load_table(table, columns_list[0])[columns_list[0]].size
                reftable_size = self.reference_storage.load_table(table, ref_columns_list[0])[ref_columns_list[0]].size
                if table_size <> reftable_size:
                    samesize = False
            if len(more) == 0 and len(less) == 0 and samesize:
                continue
            df.loc[df.shape[0]] = [table, ', '.join(less), ', '.join(more), '', 0, 0]
            if len(more) == 0 and samesize:
                continue
            # if there are columns in the "more" column, write out the corresponding years
            columns_and_years = self.cache._get_column_names_and_years(table)
            more_years = []
            for col, year in columns_and_years:
                if col in more:
                    more_years.append(year)
            df.loc[df.shape[0]-1, "Year"] = ', '.join(np.unique(np.array(more_years).astype("str")))
            if not samesize:  # there is difference in table sizes
                df.loc[df.shape[0]-1, "Size"] = table_size
                df.loc[df.shape[0]-1, "Size-ref"] = reftable_size
           
        if not check_size or (df['Size'].sum()==0 and df['Size-ref'].sum()==0):
            # remove the size columns if not used
            del df['Size']
            del df['Size-ref']
        if df.shape[0] > 0:
            logger.log_status("Differences in data structure relative to %s:" % self.reference_storage.get_storage_location())
            logger.log_status(df)
        else:
            logger.log_status("Data structure corresponds to the one in %s" % self.reference_storage.get_storage_location())
        return df
    
    def years_in_cache(self):
        return self.cache._get_sorted_list_of_years(start_with_current_year=False)