Python HDFStore.remove方法代码示例

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
 def put(self, path, obj):
     s = HDFStore(self.path)
     if path in s:
         print "updating %s" % path
         s.remove(path)
     s[path] = obj
     s.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
 def remove(self, path):
     s = HDFStore(self.path)
     if path in s:
         print("removing %s" % path)
         s.remove(path)
         s.flush(fsync=True)
     s.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
 def _put(self, path, obj):
     s = HDFStore(self.path)
     if path in s:
         print("updating %s" % path)
         s.remove(path)
         s.close()
     s = HDFStore(self.path)
     s[path] = obj
     s.flush(fsync=True)
     s.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
def main(period=None):
    temps = time.clock()
    input_tab = "C:/openfisca/output/liam/" + "LiamLeg.h5"
    output_tab = "C:/Myliam2/Model/SimulTest.h5"

    store = HDFStore(input_tab)
    goal = HDFStore(output_tab)

    name_convertion = {"ind": "person", "foy": "declar", "men": "menage", "fam": "menage"}
    # on travaille d'abord sur l'ensemble des tables puis on selectionne chaque annee
    # step 1

    for ent in ("ind", "men", "foy", "fam"):
        dest = name_convertion[ent]
        tab_in = store[ent]
        tab_out = goal["entities/" + dest]
        # on jour sur les variable a garder
        # TODO: remonter au niveau de of_on_liam mais la c'est pratique du fait de
        # l'autre table
        ident = "id" + ent
        if ent == "ind":
            ident = "noi"
        # on garde les valeurs de depart
        to_remove = [x for x in tab_in.columns if x in tab_out.columns]
        # on retire les identifiant sauf celui qui deviendra id
        list_id = ["idmen", "idfoy", "idfam", "id", "quifoy", "quifam", "quimen", "noi"]
        list_id.remove(ident)
        to_remove = to_remove + [x for x in tab_in.columns if x in list_id]
        # on n4oublie pas de garder periode
        to_remove.remove("period")
        tab_in = tab_in.drop(to_remove, axis=1)
        tab_in = tab_in.rename(columns={ident: "id"})
        tab_out = merge(tab_in, tab_out, how="right", on=["id", "period"], sort=False)
        goal.remove("entities/" + dest)
        goal.append("entities/" + dest, tab_out)
    #        new_tab = np.array(tab_out.to_records())

    store.close()
    goal.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]

#.........这里部分代码省略.........
    except:
        pdb.set_trace()
    
    # save information on qui == 0
    foy0 = table[ent].ix[table[ent]['quifoy']==0,['noi','idfoy','idmen','idfam','period']]
    men0 = table[ent].ix[table[ent]['quimen']==0,['noi','idfoy','idmen','idfam','period']]

#    # Travail sur les qui quand on ne controle pas dans la simulation que tout le monde n'est pas qui==2
## inutile car fait maintenant dans la simulation mais peut-être mieux à refaire ici un jour
## parce que ça prend du temps dans la simulation
#    time_qui = time.clock()
#    for ent in ('men','foy'): # 'fam' un jour...
#        print "Deal with qui for ", ent        
#        qui= 'qui'+ent
#        ident = 'id'+ent
#        trav = table['ind'].ix[table['ind'][qui]==2, [ident,qui,'period']]
#        for name, groupfor nom in ('menage','declar','fam'):for nom in ('menage','declar','fam'): in trav.groupby([ident,'period']):
#            to_add = range(len(group))
#            group[qui] = group[qui]+to_add
#            table['ind'].ix[group[qui].index, qui] = group[qui]
#        print "les qui pour ", ent," sont réglés"
#    time_qui = time.clock() - time_qui
#    print "le temps passé à s'occuper des qui a été",time_qui
    

    
    for entity in entities:
        nom = entity.name
        if nom in name_convertion:
            if nom != 'person': 
                pd.DataFrame(entity.array.columns)
                ent = name_convertion[nom]
                # convert from PyTables to Pandas
                table[ent] = pd.DataFrame(entity.array.columns)
                ident = 'id'+ent
                table[ent] = table[ent].rename(columns={'id': ident})
                table[ent] = merge(table[ent], eval(ent +'0'), how='left', left_on=[ident,'period'], right_on=[ident,'period'])
            # traduction de variable en OF pour ces entités
                
            if ent=='men':
                # nbinde est limité à 6 personnes et donc valeur = 5 en python
                table[ent]['nbinde'] = (table[ent]['nb_persons']-1) * (table[ent]['nb_persons']-1 <=5) +5*(table[ent]['nb_persons']-1 >5)

    table['fam'] = men0 
    
    if period is not None:
        years=[period]
        print years
    
    # a comnmenter quand on est sur du nodele pour gagner un peu de temps
#    test = {}
#    for year in years: 
#        for nom in ('menage','declar'):
#            ent = name_convertion[nom] 
##            print ent, base, ident
#            test[ent] = pd.DataFrame(entity.array.columns).rename(columns={'id': ident})
#            test[ent] = test[ent].ix[test[ent]['period']==year,:]
#            
#            test0 = eval(ent +'0')[eval(ent +'0')['period']==year]
#            
#            tab = table[ent].ix[table[ent]['period']==year,['noi','id'+ent,'idfam']]
#            ind = table['ind'].ix[table['ind']['period']==year,['qui'+ent]] 
#            try:
#                list_ind =  ind[ind==0]
#            except:
#                pdb.set_trace()            
#            lidmen = test[ent][ident]
#            lidmenU = np.unique(lidmen)
#            diff1 = set(test0[ident]).symmetric_difference(lidmenU)
#            print year, ent, diff1
#            for k in diff1:           
#    
#                pd.set_printoptions(max_columns=30)
#                listind = table['ind'][table['ind'][ident]==k]
#                print listind
#                for indiv in np.unique(listind['noi']):
#                    print table['ind'].ix[table['ind']['noi']==indiv,['noi','period','sexe','idmen','quimen','idfoy','quifoy','conj','mere','pere']]
#                    pdb.set_trace()   
              
              

    #available_years = sorted([int(x[-4:]) for x in  store.keys()])              
              
    for year in years:    
        if output=='.h5':
            try: 
                os.remove(output_tab)
            except: 
                print("Attention, la table intermediaire n'a pas ete supprimee")
            goal = HDFStore(output_tab)             
            goal.remove('survey_'+str(year))
            for ent in ('ind','men','foy','fam'):
                tab = table[ent].ix[table[ent]['period']/100==year]
                key = 'survey_'+str(year) + '/'+ent     
                goal.put(key, tab) 
            goal.close()
        else:
            for ent in ('ind','men','foy','fam'):
                table[ent] = table[ent].ix[table[ent]['period']/100==year] 
            return table       

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
 def remove(path):
     s = HDFStore(self.path)
     if path in s:
         print "removing %s" % path
         s.remove(path)
     s.close()

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]

#.........这里部分代码省略.........
## parce que ça prend du temps dans la simulation
#    time_qui = time.clock()
#    for ent in ('men','foy'): # 'fam' un jour...
#        print "Deal with qui for ", ent        
#        qui= 'qui'+ent
#        ident = 'id'+ent
#        trav = table['ind'].ix[table['ind'][qui]==2, [ident,qui,'period']]
#        for name, groupfor nom in ('menage','declar','fam'):for nom in ('menage','declar','fam'): in trav.groupby([ident,'period']):
#            to_add = range(len(group)) 
#            group[qui] = group[qui]+to_add
#            table['ind'].ix[group[qui].index, qui] = group[qui]
#        print "les qui pour ", ent," sont réglés"
#    time_qui = time.clock() - time_qui
#    print "le temps passé à s'occuper des qui a été",time_qui
    ind = table['ind']
    for ent in ['men','foy']:
        entity = _get_entity(of_name_to_til[ent])

        table[ent] = DataFrame(entity.array.columns)
        id = 'id' + ent
        qui = 'qui' + ent
        table[ent] = table[ent].rename(columns={'id': id})
        # travail sur les qui
        nb_qui = ind.loc[ind[qui]>1, ['noi',id,qui]].groupby(id, sort=True).size()
        if len(nb_qui)>0:
            new_qui = concatenated_ranges(nb_qui) + 2 
            table['ind'] = table['ind'].sort(id) #note the sort
            col_qui = table['ind'][qui]
            col_qui[col_qui>1] = new_qui
            table['ind'][qui] = col_qui 
        
        
        # informations on qui == 0
        qui0 = table['ind'].loc[table['ind']['qui' + ent]==0,['noi','idfoy','idmen','idfam','period']] 
        table[ent] = merge(table[ent], qui0, how='left', left_on=[id,'period'], right_on=[id,'period'])
    
        if ent=='men':
            # nbinde est limité à 6 personnes et donc valeur = 5 en python
            table[ent]['nbinde'] = (table[ent]['nb_persons']-1) * (table[ent]['nb_persons']-1 <=5) +5*(table[ent]['nb_persons']-1 >5)
            table['fam'] = qui0
    
    # remove non-ordinary household
    cond = (table['ind']['idmen'] >= 10) & (table['ind']['idfoy'] >= 10)
    table['ind'] = table['ind'][cond]
    table['men'] = table['men'][table['men']['idmen']>=10]
    table['foy'] = table['foy'][table['foy']['idfoy']>=10]
    table['fam'] = table['fam'][table['fam']['idfam']>=10]
    # get years
    years = np.unique(table['ind']['period'].values/100)    
    if period is not None:
        years=[period]
        print years

    if check_validity:
        for year in years: 
            ind = table['ind'] 
            for ent in ['men','foy']: #fam
                id = 'id' + ent
                qui = 'qui' + ent
                tab = table[ent]
                try:
                    assert ind.groupby([id,qui]).size().max() == 1
                except:
                    print ent
                    pb = ind.groupby([id,qui]).size() > 1
                    print(ind.groupby([id,qui]).size()[pb])
                    pdb.set_trace()
                    print(ind[ind[id]==43][['noi',id,qui]])
                
                qui0 = ind[ind[qui]==0]
                try:  
                    assert qui0[id].isin(tab[id]).all()
                except:
                    cond = tab[id].isin(qui0[id])
                    print(tab[~cond])
                    pdb.set_trace()
                try:
                    assert tab[id].isin(qui0[id]).all()
                except:
                    cond = tab[id].isin(qui0[id])
                    print(tab[~cond])
                    pdb.set_trace()

    for year in years:    
        if save_tables:
            try: 
                os.remove(output_tab)
            except: 
                print("Attention, la table intermediaire n'a pas ete supprimee")
            goal = HDFStore(output_tab)             
            goal.remove('survey_'+str(year))
            for ent in ('ind','men','foy','fam'):
                tab = table[ent].loc[table[ent]['period']/100==year]
                key = 'survey_'+str(year) + '/'+ent     
                goal.put(key, tab) 
            goal.close()
        else:
            for ent in ('ind','men','foy','fam'):
                table[ent] = table[ent].loc[table[ent]['period']/100==year] 
            return table       

# 需要导入模块: from pandas import HDFStore [as 别名]
# 或者: from pandas.HDFStore import remove [as 别名]
# il y a un truc avec les gens qui se marient puis divorcent
# en profiter pour bien gerer les conj = 0 ou conj =-1
        # si on ne s'arrete pas là, c'est qu'on n'a pas de problème !! 
        print year, ent, diff1
        for k in diff1:           

            pd.set_printoptions(max_columns=30)
            listind = table['ind'][table['ind'][ident]==k]
            print listind
            for indiv in np.unique(listind['id']):
                print table['ind'].ix[table['ind']['id']==indiv,['id','period','sexe','idmen','quimen','idfoy','quifoy','conj','mere','pere']]
                pdb.set_trace()   
        
            
for year in years:
    goal.remove('survey_'+str(year))
    for ent in ('ind','men','foy','fam'):
        tab = table[ent].ix[table[ent]['period']==year]
        key = 'survey_'+str(year) + '/'+ent     
        goal.put(key, tab) 
#    if year == 2010:
#        pdb.set_trace()
#        tab = table[ent].ix[table[ent]['period']==year]
#        tab[:5]
#        len(tab['idfam'])
#        len(np.unique(tab['idfam']))
#        list_qui = tab['idfam']
#        double = list_qui.value_counts()[list_qui.value_counts()>1]
#        tabind = table['ind'].ix[table['ind']['period']==year]