Python HDFStore.append方法代码示例

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def storeHdf5(data, tag, path):
    hdf = HDFStore(path,'a')
    if tag in hdf.keys():
        hdf.append(tag,data)
    else:
        hdf.put(tag,data)
    hdf.close()          

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
class HDFStorePanel(BaseIO):

    goal_time = 0.2

    def setup(self):
        self.fname = '__test__.h5'
        with warnings.catch_warnings(record=True):
            self.p = Panel(np.random.randn(20, 1000, 25),
                           items=['Item%03d' % i for i in range(20)],
                           major_axis=date_range('1/1/2000', periods=1000),
                           minor_axis=['E%03d' % i for i in range(25)])
            self.store = HDFStore(self.fname)
            self.store.append('p1', self.p)

    def teardown(self):
        self.store.close()
        self.remove(self.fname)

    def time_read_store_table_panel(self):
        with warnings.catch_warnings(record=True):
            self.store.select('p1')

    def time_write_store_table_panel(self):
        with warnings.catch_warnings(record=True):
            self.store.append('p2', self.p)

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
    def store_results(self, result, index, columns, hdf5_file):
        self.df = DataFrame(result, columns=columns)
        self.df = self.df.set_index(index)
        self.df.sort_index(inplace=True)

        # Store the DataFrame as an HDF5 file...
        hdf = HDFStore(hdf5_file)
        # Append the dataframe, and ensure addr / host can be 17 chars long
        hdf.append('df', self.df, data_columns=list(columns), 
            min_itemsize={'addr': 17, 'host': 17})
        hdf.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
    def store_to_liam(self):
        '''
        Sauvegarde des données au format utilisé ensuite par le modèle Til
        Séléctionne les variables appelée par Til derrière
        Appelle des fonctions de Liam2
        '''
        path = self._output_name()
        h5file = tables.openFile(path, mode="w")

        ent_node = h5file.createGroup("/", "entities", "Entities")
        for ent_name in ['ind', 'foy', 'men', 'futur', 'past']:
            entity = eval('self.' + ent_name)
            if entity is not None:
                entity = entity.fillna(-1)
                try:
                    ent_table = entity.to_records(index=False)
                except:
                    pdb.set_trace()
                dtypes = ent_table.dtype
                final_name = of_name_to_til[ent_name]
                try:
                    table = h5file.createTable(ent_node, final_name, dtypes, title="%s table" % final_name)
                    table.append(ent_table)
                except:
                    pdb.set_trace()
                table.flush()

                if ent_name == 'men':
                    entity = entity.loc[entity['id']>-1]
                    ent_table2 = entity[['pond', 'id', 'period']].to_records(index=False)
                    dtypes2 = ent_table2.dtype
                    table = h5file.createTable(ent_node, 'companies', dtypes2, title="'companies table")
                    table.append(ent_table2)
                    table.flush()
                if ent_name == 'ind':
                    ent_table2 = entity[['agem', 'sexe', 'pere', 'mere', 'id', 'findet', 'period']].to_records(
                        index = False)
                    dtypes2 = ent_table2.dtype
                    table = h5file.createTable(ent_node, 'register', dtypes2, title="register table")
                    table.append(ent_table2)
                    table.flush()
        h5file.close()

        # 3 - table longitudinal
        # Note: on conserve le format pandas ici
        store = HDFStore(path)
        for varname, table in self.longitudinal.iteritems():
            table['id'] = table.index
            store.append('longitudinal/' + varname, table)
        store.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def pf2pandas(wd, files, vars=None, npwd=None, rmvars=None,   \
            debug=False):
    """ 
    Read in GEOS-Chem planeflight output and convert to HDF format

     - Converts date and time columns to datetime format indexes
     - the resultant HDF is in 2D list form 
    ( aka further processing required to 3D /2D output  )
        
    Note: 
     - This function is limited by the csv read speed. for large csv output expect 
     significant processing times or set to automatically run post run
     - Original files are not removed, so this function will double space usage for 
     output unless the original fiels are deleted.
    """

    # Ensure working dorectory string has leading foreward slash
    if wd[-1] != '/':
        wd += '/'

#    pfdate =( re.findall('\d+', file ) )[-1]
    if not isinstance(vars, list ):
        vars, sites = get_pf_headers( files[0], debug=debug )
    if not isinstance(npwd, str ):
        npwd = get_dir('npwd')
    hdf =HDFStore( npwd+ 'pf_{}_{}.h5'.format( wd.split('/')[-3], \
        wd.split('/')[-2], wd.split('/')[-1]  ))
    
    if debug:
        print hdf

    for file in files:
        print file#, pfdate

        # convert planeflight.log to DataFrame
        df = pf_csv2pandas( file, vars )
            
        if file==files[0]:
            hdf.put('d1', df, format='table', data_columns=True)
        else:
            hdf.append('d1', df, format='table', data_columns=True)

        if debug:
            print hdf['d1'].shape, hdf['d1'].index
        del df
    hdf.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
 def to_frame_hdf(self, store_path, store_key, df_cb=None, max_msg=None,
                  usecols=None, chunk_cnt=CHUNK_CNT):
     """Convert to Pandas DataFrame and save to HDF then returns
     HDFStore."""
     store = HDFStore(store_path, 'w')
     _c = self._to_frame_prop('to_frame_hdf', False)
     for df in self._to_frame_gen(_c, usecols, chunk_cnt):
         min_itemsize = {'kind': 20, 'msg': 255}
         # pytables not support unicode for now
         df['msg'] = df['msg'].apply(lambda m: m.encode('utf8'))
         if df_cb is not None:
             df_cb(df)
         if max_msg is not None:
             min_itemsize['msg'] = max_msg
         store.append(store_key, df, format='table',
                      min_itemsize=min_itemsize)
     store.flush()
     store.close()
     _c.pg.done()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def csv2hdf5(csv_name, h5_name, dfname, option='frame'):
    """
    Convert a csv file to a dataframe in a hdf5

    Parameters:

    csv_name: string
              csv file name
    h5_name : string
              hdf5 file name
    dfname  : string
              dataframe name
    option  : string, 'frame' or 'table', default to 'frame'
              stoing type in the pytable
    """

    table = read_csv(csv_name)
    store = HDFStore(h5_name)

    if option == 'frame':
        store.put(dfname, table)

    elif option == 'table': # for frame_table à la pytables
        object_cols =  table.dtypes[ table.dtypes == 'object']
        print object_cols.index
        try:
            store.append(dfname,table)
        except:
            print table.get_dtype_counts()
            object_cols =  table.dtypes[ table.dtypes == 'object']

            for col in object_cols.index:
                print 'removing object column :', col
                del table[col]

            store.append(dfname,table)

    print store
    store.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def main(period=None):
    temps = time.clock()
    input_tab = "C:/openfisca/output/liam/" + "LiamLeg.h5"
    output_tab = "C:/Myliam2/Model/SimulTest.h5"

    store = HDFStore(input_tab)
    goal = HDFStore(output_tab)

    name_convertion = {"ind": "person", "foy": "declar", "men": "menage", "fam": "menage"}
    # on travaille d'abord sur l'ensemble des tables puis on selectionne chaque annee
    # step 1

    for ent in ("ind", "men", "foy", "fam"):
        dest = name_convertion[ent]
        tab_in = store[ent]
        tab_out = goal["entities/" + dest]
        # on jour sur les variable a garder
        # TODO: remonter au niveau de of_on_liam mais la c'est pratique du fait de
        # l'autre table
        ident = "id" + ent
        if ent == "ind":
            ident = "noi"
        # on garde les valeurs de depart
        to_remove = [x for x in tab_in.columns if x in tab_out.columns]
        # on retire les identifiant sauf celui qui deviendra id
        list_id = ["idmen", "idfoy", "idfam", "id", "quifoy", "quifam", "quimen", "noi"]
        list_id.remove(ident)
        to_remove = to_remove + [x for x in tab_in.columns if x in list_id]
        # on n4oublie pas de garder periode
        to_remove.remove("period")
        tab_in = tab_in.drop(to_remove, axis=1)
        tab_in = tab_in.rename(columns={ident: "id"})
        tab_out = merge(tab_in, tab_out, how="right", on=["id", "period"], sort=False)
        goal.remove("entities/" + dest)
        goal.append("entities/" + dest, tab_out)
    #        new_tab = np.array(tab_out.to_records())

    store.close()
    goal.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
class HDFStoreDataFrame(BaseIO):

    def setup(self):
        N = 25000
        index = tm.makeStringIndex(N)
        self.df = DataFrame({'float1': np.random.randn(N),
                             'float2': np.random.randn(N)},
                            index=index)
        self.df_mixed = DataFrame({'float1': np.random.randn(N),
                                   'float2': np.random.randn(N),
                                   'string1': ['foo'] * N,
                                   'bool1': [True] * N,
                                   'int1': np.random.randint(0, N, size=N)},
                                  index=index)
        self.df_wide = DataFrame(np.random.randn(N, 100))
        self.start_wide = self.df_wide.index[10000]
        self.stop_wide = self.df_wide.index[15000]
        self.df2 = DataFrame({'float1': np.random.randn(N),
                              'float2': np.random.randn(N)},
                             index=date_range('1/1/2000', periods=N))
        self.start = self.df2.index[10000]
        self.stop = self.df2.index[15000]
        self.df_wide2 = DataFrame(np.random.randn(N, 100),
                                  index=date_range('1/1/2000', periods=N))
        self.df_dc = DataFrame(np.random.randn(N, 10),
                               columns=['C%03d' % i for i in range(10)])

        self.fname = '__test__.h5'

        self.store = HDFStore(self.fname)
        self.store.put('fixed', self.df)
        self.store.put('fixed_mixed', self.df_mixed)
        self.store.append('table', self.df2)
        self.store.append('table_mixed', self.df_mixed)
        self.store.append('table_wide', self.df_wide)
        self.store.append('table_wide2', self.df_wide2)

    def teardown(self):
        self.store.close()
        self.remove(self.fname)

    def time_read_store(self):
        self.store.get('fixed')

    def time_read_store_mixed(self):
        self.store.get('fixed_mixed')

    def time_write_store(self):
        self.store.put('fixed_write', self.df)

    def time_write_store_mixed(self):
        self.store.put('fixed_mixed_write', self.df_mixed)

    def time_read_store_table_mixed(self):
        self.store.select('table_mixed')

    def time_write_store_table_mixed(self):
        self.store.append('table_mixed_write', self.df_mixed)

    def time_read_store_table(self):
        self.store.select('table')

    def time_write_store_table(self):
        self.store.append('table_write', self.df)

    def time_read_store_table_wide(self):
        self.store.select('table_wide')

    def time_write_store_table_wide(self):
        self.store.append('table_wide_write', self.df_wide)

    def time_write_store_table_dc(self):
        self.store.append('table_dc_write', self.df_dc, data_columns=True)

    def time_query_store_table_wide(self):
        self.store.select('table_wide', where="index > self.start_wide and "
                                              "index < self.stop_wide")

    def time_query_store_table(self):
        self.store.select('table', where="index > self.start and "
                                         "index < self.stop")

    def time_store_repr(self):
        repr(self.store)

    def time_store_str(self):
        str(self.store)

    def time_store_info(self):
        self.store.info()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def extract_relevant_data( case_list = [], exceptions = [], y_delta_locs = [],
                         x_2h_locs = [] , plot = False):
    """ This will extract the wall normal data at the spanwise location
    TE at a certain y density
    """

    from os                           import listdir
    from os.path                      import join,split
    from pandas                       import DataFrame, HDFStore, read_pickle
    from boundary_layer_routines      import return_bl_parameters
    from raw_data_processing_routines import decript_case_name
    from progressbar                  import ProgressBar,Percentage
    from progressbar                  import Bar,ETA,SimpleProgress
    from numpy                        import array, round, linspace
    from data_cleaning_routines       import show_surface_from_df

    x_2h_locs    = round( array( x_2h_locs ),    2 )
    y_delta_locs = round( array( y_delta_locs ), 2 )

    # Get the available HDF5 files #############################################
    hdf5_root = '/media/carlos/6E34D2CD34D29783/' +\
                '2015-02_SerrationPIV/TR_Data_Location_Calibrated_Article3'

    if not len(case_list):
        hdf5_files = [f for f in listdir( hdf5_root ) \
                      if f.endswith('.hdf5') \
                      and not f in exceptions ]
    else:
        hdf5_files = [f for f in listdir( hdf5_root ) \
                      if f.endswith('.hdf5') \
                      and f in case_list ]
    # ##########################################################################

    for hf in [join( hdf5_root, f ) for f in hdf5_files]:

        f = split( hf )[1].replace('_AirfoilNormal','')\
                .replace('_Aligned.hdf5','')

        print "   Extracting data from {0}".format(f)
        print "     at the normalized streamwise locations:"
        print "     {0}".format( x_2h_locs )


        hdf_t = HDFStore( hf, 'r' )

        # Get the available coordinates ########################################
        hf_coords = hdf_t.select('data', where = [ 't = 0' ], 
                                 columns = [ 'x', 'y' ] )
        # ######################################################################

        # Turn the non-dim requested locations into physical coords ############
        requested_locations = []
        requested_normalized_locations = []
        #for x,x_norm in zip(x_2h_locs * tooth_length, x_2h_locs):
        #    for y_d in y_delta_locs:
        #        bl_params = return_bl_parameters( f , [x] )
        #        d_99 = bl_params.delta_99.values[0]
        #        #if "STE" in f:
        #        #    d_99 = 9.4
        #        y = y_d * d_99
        #        requested_locations.append( (x,y) )
        #        requested_normalized_locations.append( ( x_norm, y_d ) )

        # Get the normalization locations depending on the case ################
        if 'z00' in f and not 'STE' in f:
            x_bl_loc = 40
        elif 'z05' in f:
            x_bl_loc = 20
        elif 'z10' in f or 'STE' in f:
            x_bl_loc = 0

        bl_params = return_bl_parameters( f , [x_bl_loc] )
        d_99 = bl_params.delta_99.values[0]

        for x,x_norm in zip(x_2h_locs * tooth_length, x_2h_locs):
            for y_d in y_delta_locs:
                y = y_d * d_99
                requested_locations.append( (x,y) )
                requested_normalized_locations.append( ( x_norm, y_d ) )
        print "    Normalizing to a BL thickness of {0:.2f} mm".\
                format(d_99)
        # ######################################################################

        available_xy_locs = hf_coords[
            ( hf_coords.x > min( x_2h_locs ) * 40. ) & \
            ( hf_coords.x < max( x_2h_locs ) * 40. ) & \
            ( hf_coords.y > min( y_delta_locs ) * d_99 ) & \
            ( hf_coords.y < max( y_delta_locs ) * d_99 )
        ][ ['x','y'] ]
              
        available_xy_locs = [tuple(x) for x in available_xy_locs.values]

        if plot:

            trailing_edge,phi,alpha,U,z = decript_case_name( f )

            if trailing_edge == 'serrated': device = 'Sr20R21'
            elif trailing_edge == 'straight': device = 'STE'
            elif trailing_edge == 'slitted': device = 'Slit20R21'

#.........这里部分代码省略.........

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
def read_raw_tecplot_folder_and_write_pandas_hdf5(
    case_folder,
    root                  = 0,
    output_file           = 0,
    output_root           = 0,
    overwrite             = False,
):
    from os.path     import isfile,join,splitext
    from os          import listdir
    from progressbar import ProgressBar,Percentage,Bar
    from progressbar import ETA,SimpleProgress
    from pandas      import DataFrame, HDFStore

    # File related things ######################################################
    if not output_file:
        output_file = case_folder+"_Aligned.hdf5"

    if not output_root:
        output_root = '/media/carlos/6E34D2CD34D29783/' +\
                '2015-02_SerrationPIV/TR_Data_Location_Calibrated_Article3'

    if not output_file.endswith('_Aligned.hdf5'):
        output_file = output_file.replace("_Aligned.hdf5","")+"_Aligned.hdf5"

    if 'STE' in case_folder or 'z10' in case_folder:
        output_file = output_file.replace( '.hdf5', '_AirfoilNormal.hdf5' )

    if isfile(join( output_root, output_file )) and not overwrite:
        print "  Exiting; file exists:\n      {0}".format(output_file)
        return 0
    else:
        print "  Writing\n      {0}".format(output_file)

    # ##########################################################################


    time_step_files = sorted(
        [join(root,case_folder,f) for f in listdir(join( root, case_folder )) \
         if splitext(f)[1] == '.dat']
    )

    progress = ProgressBar(
         widgets=[
             Bar(),' ',
             Percentage(),' ',
             ETA(), ' (file ',
             SimpleProgress(),')'], 
         maxval=len(time_step_files)
         ).start()

    cnt = 0

    hdf_store = HDFStore( join( output_root, output_file ) )

    for f,t in zip(time_step_files,range(len(time_step_files))):

       df_t = read_tecplot_file(
           tecplot_folder         = join( root, case_folder ),
           tecplot_time_step_file = f,
           time_step              = t,
       )

       if cnt == 0:
           df = df_t.copy()
       else:
           df = df.append( df_t, ignore_index = True)

       if cnt == 50:

           df = correct_df_translation_rotation( df )\
                   [['x','y','t','u','v','w']]

           df = df.sort_values( by = ['x','y','t'] )

           #df.set_index( ['x','y'], inplace = True)

           if t == 0:
               hdf_store.put( 'data', df , 
                                data_columns = ['x','y','t'],
                               format = 't')
           else:
               hdf_store.append( 'data', df , 
                                data_columns = ['x','y','t'],
                               format = 't')

           cnt = 0

           df = DataFrame()

       cnt += 1

       progress.update(t)

    progress.finish()

    hdf_store.close()

    return 1

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
projection = HDFStore(
    "C:\Users\Utilisateur\Documents\GitHub\ga\src\countries\France\sources\data_fr\proj_pop_insee\proj_pop.h5", "r"
)
projection_dataframe = projection[
    "/projpop0760_FECbasESPbasMIGbas"
]  # <-Do not know the precise meaning of this. For testing only

# Combining
concatened = concat([population, projection_dataframe], verify_integrity=True)
concatened = concatened.reset_index()
concatened["year"] = concatened.year.convert_objects(convert_numeric=True)
concatened = concatened.set_index(["age", "sex", "year"])

# Saving as HDF5 file
export = HDFStore("neo_population.h5")
export.append("pop", concatened, data_columns=concatened.columns)
export.close()
export = HDFStore("neo_population.h5", "r")
print export


# Creating the simulation object
net_payments = Simulation()
net_payments.set_population(population)

France = "France"
net_payments.set_country(France)
r = 0.0
g = 0.01
net_payments.set_discount_rate(r)
net_payments.set_growth_rate(g)

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import append [as 别名]
    def store_to_liam(self):
        '''
        Sauvegarde des données au format utilisé ensuite par le modèle Til
        Séléctionne les variables appelée par Til derrière
        Appelle des fonctions de Liam2
        '''

        path_param = os.path.join(path_model, "til_base_model\param", "globals.csv")
        path = os.path.join(path_model, self._output_name())
        h5file = tables.openFile( path, mode="w")
        # 1 - on met d'abord les global en recopiant le code de liam2
        # globals_def = {'periodic': {'path': 'param/globals.csv'}}
        globals_def = {'periodic': {'path': path_param}}

        const_node = h5file.createGroup("/", "globals", "Globals")
        localdir = path_model
        for global_name, global_def in globals_def.iteritems():
            print(" %s" % global_name)
            req_fields = ([('PERIOD', int)] if global_name == 'periodic'
                                            else [])
            kind, info = imp.load_def(localdir, global_name,
                                  global_def, req_fields)
            # comme dans import
#             if kind == 'ndarray':
#                 imp.array_to_disk_array(h5file, const_node, global_name, info,
#                                     title=global_name,
#                                     compression=compression)
#             else:
            assert kind == 'table'
            fields, numlines, datastream, csvfile = info
            imp.stream_to_table(h5file, const_node, global_name, fields,
                            datastream, numlines,
                            title="%s table" % global_name,
                            buffersize=10 * 2 ** 20,
                            compression=None)

        # 2 - ensuite on s'occupe des entities
        ent_node = h5file.createGroup("/", "entities", "Entities")
        for ent_name in ['ind','foy','men','futur','past']:
            entity = eval('self.'+ ent_name)
            if entity is not None:
                entity = entity.fillna(-1)
                ent_table = entity.to_records(index=False)
                dtypes = ent_table.dtype
                final_name = of_name_to_til[ent_name]
                table = h5file.createTable(ent_node, final_name, dtypes, title="%s table" % final_name)
                table.append(ent_table)
                table.flush()

                if ent_name == 'men':
                    entity = entity.loc[entity['id']>-1]
                    ent_table2 = entity[['pond','id','period']].to_records(index=False)
                    dtypes2 = ent_table2.dtype
                    table = h5file.createTable(ent_node, 'companies', dtypes2, title="'companies table")
                    table.append(ent_table2)
                    table.flush()
                if ent_name == 'ind':
                    ent_table2 = entity[['agem','sexe','pere','mere','id','findet','period']].to_records(index=False)
                    dtypes2 = ent_table2.dtype
                    table = h5file.createTable(ent_node, 'register', dtypes2, title="register table")
                    table.append(ent_table2)
                    table.flush()
        h5file.close()

        # 3 - table longitudinal
        # Note: on conserve le format pandas ici
        store = HDFStore(path)
        for varname, tab in self.longitudinal.iteritems():
            #format to liam
            table = tab
            table['id'] = table.index

            store.append('longitudinal/' + varname, table)
        store.close()