Python tabular.tabarray函数代码示例

def specific_config_gen(IC,args):
    IC.base_dir = args['base_dir']
    IC.annotate_dir = args['annotate_dir']
    IC.groundtruth_dir = args['groundtruth_dir']
    IC.correspondence = tb.tabarray(SVfile = args['frame_correspondence'])
    IC.size = args['size']
    IC.prefix = prefix = args.get('image_extension','.jpg')
    IC.current_frame_path = None
    csvs = [x for x in os.listdir(IC.annotate_dir) if x.endswith('.csv')]
    csvs.sort()
    Xs = [tb.tabarray(SVfile = os.path.join(IC.annotate_dir,csv)) for csv in csvs]
    cns = [csv.split('.')[0] for csv in csvs]
    cns = [[cn]*len(X) for (cn,X) in zip(cns,Xs)]
    Xs = [X.addcols(cn,names=['clip_num']) for (cn,X) in zip(cns,Xs)]

    csvs = [x for x in os.listdir(IC.groundtruth_dir) if x.endswith('.csv')]
    csvs.sort()
    Gs = []
    fields = ['clip_num','Frame'] + xfields + yfields
    for ind,csv in enumerate(csvs):
        try:
            g = tb.tabarray(SVfile = os.path.join(IC.groundtruth_dir,csv))
        except:
            x = Xs[ind].addcols([-1]*len(Xs[ind]),names=['Correctness'])
        else:
            g = g.addcols([csv.split('.')[0]]*len(g),names = ['clip_num'])
            g = g[fields + ['Confidence']]
            g.renamecol('Confidence','Correctness')
            x = Xs[ind].join(g,keycols=fields)
        Gs.append(x)
    X = tb.tab_rowstack(Gs)
    X.sort(order=['clip_num','Frame'])
    
    Y = IC.correspondence
    F = tb.fast.recarrayisin(Y[['clip_num','Frame']],X[['clip_num','Frame']])
    Y = Y[F]
    X = X.join(Y,keycols=['clip_num','Frame'])

    params = []
    for t in X:
        print(t)  
        cn = t['clip_num']
        fr = t['Frame']
        box = get_darpa_box(t)
        bb = box.pop('box')
        xc,yc = bb.center
        center = correct_center((xc,yc),IC.size,(1920,1080))
        bb_new = bbox.BoundingBox(center = center,width = IC.size[0], height = IC.size[1])
        p = SON([('size',IC.size),
                     ('bounding_box',SON([('xfields',list(bb_new.xs)),('yfields',list(bb_new.ys))])),
                     ('original_bounding_box',SON([('xfields',list(bb.xs)),('yfields',list(bb.ys))])),
                     ('clip_num',cn),
                     ('Frame',int(t['Original'])),
                     ('base_dir',IC.base_dir),
                     ('correctness',int(t['Correctness']))])
        p.update(box)
        p['GuessObjectType'] = p['ObjectType']
        p['ObjectType'] = p['ObjectType'] if t['Correctness'] == 1 else ''
        params.append(SON([('image',p)]))
    return params

def TestPivot2():
    X = tb.tabarray(records=[('x', 1, 3, 6), ('y', 0, 3, 1), ('x', 0, 3, 5)], 
                    names=['a', 'b', 'c', 'd'])
    Y = X.pivot('b', 'a')
    Z = tb.tabarray(records=[(0, 3, 3, 5, 1), (1, 3, 0, 6, 0)], 
                    names=['b', 'x_c', 'y_c', 'x_d', 'y_d'])
    assert (Y == Z).all()

def make_background_db(
    creates="../background_certificate.txt", depends_on=("../3d_hdr_backgrounds.csv", "../2d_grayscale_backgrounds.csv")
):

    conn = pm.Connection()
    db = conn["dicarlocox_3dmodels"]
    db.drop_collection("3d_spherical_backgrounds")

    coll = db["3d_spherical_backgrounds"]

    recs = [
        {"name": "backlot", "path": "backlot.tdl"},
        {"name": "apartment", "path": "apartment.tdl"},
        {"name": "empty room", "path": "empty_room.tdl"},
        {"name": "office", "path": "office.tdl"},
    ]

    for rec in recs:
        coll.insert(rec)

    X = tb.tabarray(SVfile=depends_on[0])
    recs = [{"name": x["Path"][:-4], "path": x["Path"], "description": x["Description"], "type": "3d hdr"} for x in X]
    for rec in recs:
        coll.insert(rec)

    X = tb.tabarray(SVfile=depends_on[1])
    recs = [{"name": x["Path"][:-4], "path": x["Path"], "type": "2d grayscale"} for x in X]
    for rec in recs:
        coll.insert(rec)

    rec = {"name": "blank gray image", "path": "gray.td", "type": "blank"}
    coll.insert(rec)

def tabularmetadataforms(pathlist,depends_on = '../System/MetaData/'):
	attlist = ['description','author','title','keywords']
	recs1 = []
	recs2 = []
	for x in pathlist:
		print x
		mdp = metadatapath(x) + '/ProcessedMetaData.pickle'
		if PathExists(mdp):
			M = pickle.load(open(mdp))
			D = {}
			for att in attlist:
				if att in M.keys():
					D[att] = M[att]
				else:
					D[att] = ''
			recs1.append((x,) + tuple([D[att].replace('\n',' ') for att in attlist]))
			colnames = M['colnames']
			if 'coldescrs' in M.keys():
				coldescrs = [M['coldescrs'][m] if m in M['coldescrs'].keys() else ''  for m in colnames]
			else:
				coldescrs = ['']*len(colnames)
			
			recs2 += zip([x]*len(colnames),colnames,coldescrs)		
		
	X = tb.tabarray(records = recs1,names=['Path'] + attlist)
	Y = tb.tabarray(records = recs2,names = ['Path','ColName','ColDescr'])
	
	return [X,Y]

def _normalize_array(x):
    flour = tb.tabarray(columns=[x['flour_all'] + x['flour_almond'] + \
            x['flour_bread'] + x['flour_cake'] + x['flour_other'] + \
            x['flour_rice'] + x['flour_wheat'] + x['flour_rye']], \
            names=['flour'])

    liquid = tb.tabarray(columns=[x['milk'] + x['water'] + \
            x['soymilk'] + x['buttermilk'] + x['juice_apple'] + \
            x['juice_can'] + x['juice_lemon'] + x['juice_lime'] + \
            x['juice_orange'] + x['juice_other'] + x['juice_pineapple']], \
            names=['liquid'])

    fat = tb.tabarray(columns=[x['butter'] + x['cream_cheese'] + \
            x['cream_half'] + + x['cream_ice'] + x['cream_other'] + \
            x['cream_tartar'] + x['cream_whipped'] + x['margarine'] + \
            x['oil_canola'] + x['oil_olive'] + x['oil_other'] + \
            x['oil_vegetable'] + x['lard'] + x['shortening']], \
            names=['fat'])

    sugar = tb.tabarray(columns=[x['sugar_brown'] + x['sugar_powder'] + \
            x['sugar_white']], names=['sugar'])

    a = x[['egg']].colstack(fat).colstack(flour).colstack(liquid)\
            .colstack(sugar).extract()
    a = a / np.repeat(a.sum(axis = 1), a.shape[1])\
            .reshape(a.shape[0], a.shape[1])
    return a

def test_bionumbers():
    X = tb.tabarray(SVfile = 'tests/bionumbers.txt') 
    fname = TestDataDir + 'bionumbers.txt'
    X.saveSV(fname, quotechar="'")
    Y = tb.tabarray(SVfile = TestDataDir + 'bionumbers.txt',quotechar="'")
    names = ('ID', 'Property', 'Organism', 'Value', 'Units', 'Range', 
              'NumericalValue', 'Version')               
    assert_bio(X.dtype.names == names and len(X) == 4615 and eq(X,Y), fname)

 def test_missingvals4(self):
     fname = TestDataDir + 'missingvals4.csv'
     F = open(fname,'w')
     F.write('Name,Age,Gender\nDaniel,12,M\nElaine,'',F\nFarish,46,')
     F.close()
     X = tb.tabarray(SVfile=fname)
     X2 = tb.tabarray(records=[('Daniel', 12, 'M'), ('Elaine', np.nan, 'F'), ('Farish', 46, '')],names=['Name','Age','Gender'])
     self.assert_io(eq(X, X2), fname)                        

def makemetadata(code,datadir,outfile1,outfile2,depends_on = (resource_root + 'ProcessedManifest_2_HandAdditions.tsv',resource_root + 'Keywords.txt')):

    Z  = {}

    keyword_file = depends_on[1]
    Y = tb.tabarray(SVfile = keyword_file)[['Code','Keywords']]
    y = Y[Y['Code'] == code]
    Z['keywords'] = [x.strip() for x in str(y['Keywords'][0]).split(',')]
    
    
    dirl = np.array(listdir(datadir))
    
    pr = lambda x : x.split('!')[-1][:-4]
    p=re.compile('\([^\)]*\)')
    
    tps = [l for l in dirl if l.endswith('.txt.txt')]
    if tps:
        textpath = datadir + tps[0]
        [SD,things] = ParseTexts(textpath,code)
        FNs = [p.sub('',things[pr(y).lower()]).replace(' ,',',').replace(',,',',') if pr(y).lower() in things.keys() else '' for y in dirl]
        FNs = [z.split('=')[1] if '=' in z and not ' =' in z else z for z in FNs]
    else:
        SD = ''
        FNs = len(dirl)*['']
        
    Z['description'] = SD
    
    cfs = [l for l in dirl if l.endswith('.contacts.txt')]
    if cfs:
        contactfile = datadir + cfs[0]
        ctext = open(contactfile,'rU').read().strip()
        if '<html>' in ctext.lower():
            clines = ctext.split('\n')
            fb = [i for i in range(len(clines)) if clines[i].strip() == ''][0]
            ctext = '\n'.join(clines[fb+1:])
        ctext = ctext.strip(' *\n').replace('\n\n','\n')    
    else:
        ctext = ''
        
    Z['contactInfo'] = ctext
    f = open(outfile1,'w')
    pickle.dump(Z,f)
    f.close()

    Y = tb.tabarray(SVfile = depends_on[0])
    Y.sort(order = ['File'])

    
    dirlp = np.array([pr(y) for y in dirl])
    [A,B] = tb.fast.equalspairs(dirlp,Y['File'])
    if (B>A).any():
        print 'adding hand-made content to', dirlp[B>A]
        for k in (B>A).nonzero()[0]:
            FNs[k] = Y['FileName'][A[k]]    
    
    D = tb.tabarray(columns=[dirl,FNs], names = ['Path','FileName'])
    
    D.saveSV(outfile2,metadata = True)  

def TestReplace2():
    V1 = ['North', 'South', 'East', 'West']
    V2 = ['Service', 'Manufacturing', 'Education', 'Healthcare']
    Recs = [(a, b, np.random.rand() * 100, np.random.randint(100000)) 
                                                       for a in V1 for b in V2]
    X = tb.tabarray(records=Recs, 
                    names=['Region', 'Sector', 'Amount', 'Population'])
    X2 = tb.tabarray(records=Recs, 
                     names=['Region', 'Sector', 'Amount', 'Population'])

    X.replace('S', 'M')
    assert((X == X2).all())

def get_subclass_pages():
    X = tb.tabarray(SVfile = 'catlevels.tsv')
    recs = []
    p = re.compile('Sub\d')
    f = lambda x : p.match(dict(x.attrs).get('class',''))
    for x in X[:]:
        subrecs = []
        cat = x['CLASS']
        fixed_cat = fix_cat(cat)
        title = x['CLASS TITLE']
        os.system('wget http://www.uspto.gov/web/patents/classification/uspc' + fixed_cat + '/sched' + fixed_cat + '.htm -O ' + cat + '.html')
        Soup = BeautifulSoup.BeautifulSoup(open(cat + '.html'))
        Crac = Soup.find(True,'CracHeader')
        For = Soup.find(True,'ForHeader')
        Dig = Soup.find(True,'DigHeader')
        if Crac:
            end = Crac
        elif For:
            end = For
        elif Dig:
            end = Dig
        else:
            end  = None
        if end:
            T = end.findAllPrevious('tr',valign='top')[:]
        else:
            T = Soup.findAll('tr',valign='top')[:]
        T.reverse()
        for (i,t) in enumerate(T): 
            try:
                subclass = str(Contents(t.find(f)).replace(' ','').strip())
            except:
                pass
            else:
                try:
                    subtitle = Contents(t.find(True,"SubTtl")).strip()
                except:
                    pass
                else:
                    try:
                        indent = int(dict(t.find(True,"SubTtl").find("img").attrs)['src'].split('/')[-1].split('_')[0])
                    except AttributeError:
                        indent = 0
                    #print (cat,title,subclass,subtitle,indent)    
                    subrecs.append((cat,title,subclass,subtitle,indent))
        subrecs.reverse()
        recs.extend(subrecs)

    Y = tb.tabarray(records = recs, names=['Class','Title','Subclass','Subtitle','Indent'],formats=['str','str','str','str','int'])
    Y.saveSV('classifications.tsv',metadata=True)

def TestPivot4():
    V1 = ['NorthAmerica', 'SouthAmerica', 'Europe', 'Asia', 'Australia', 'Africa', 'Antarctica']
    V1.sort()
    V2 = ['House', 'Car', 'Boat', 'Savings', 'Food', 'Entertainment', 'Taxes']
    V2.sort()
    Recs = [(a, b, 100 * np.random.rand()) for a in V1 for b in V2]
    X = tb.tabarray(records=Recs[:-1],
                    names=['Region', 'Source', 'Amount'])
    Y = X.pivot('Region', 'Source', 
                NullVals=dict([(o,-999) for o in X.dtype.names]))
    X2 = tb.tabarray(records=Recs, names=['Region', 'Source', 'Amount'])
    Y2 = X.pivot('Region','Source')
    Y2[V2[-1] + '_Amount'][-1] = -999

    assert (Y == Y2).all()

def TestPivot3():
    V1 = ['NorthAmerica', 'SouthAmerica', 'Europe', 'Asia', 'Australia', 
          'Africa', 'Antarctica']
    V1.sort()
    V2 = ['House', 'Car', 'Boat', 'Savings', 'Food', 'Entertainment', 'Taxes']
    V2.sort()
    Recs = [(a, b, 100 * np.random.rand()) for a in V1 for b in V2]
    X = tb.tabarray(records=Recs, names=['Region', 'Source', 'Amount'])
    Y = X.pivot('Region', 'Source')
    Z = utils.uniqify(X['Source'])
    Z.sort()
    Cols = [[y['Amount'] for y in X if y['Source'] == b] for b in Z]
    W = tb.tabarray(columns=[V1] + Cols, 
                    names=['Region'] + [b + '_Amount' for b in Z])
    assert (W == Y).all()

    def get_data(self):

        trial_records = []
        obj_oi = np.unique(self.meta["obj"])
        img_oi = np.unique(self.meta["id"])

        for subj in self.data:
            for r, i, ss in zip(subj["Response"], subj["ImgData"], subj["StimShown"]):
                if len(i) > 1:
                    s = i["Sample"]
                    t = i["Test"]
                    s_id = s["id"]
                    s_obj = s["obj"]
                    t_obj = [t_["obj"] for t_ in t]
                    d_obj = [t_ for t_ in t_obj if t_ != s_obj][0]
                    resp = t_obj[r]
                else:  # backwards compatibility with previous mturkutils
                    s_id = i[0]["id"]
                    s_obj = i[0]["obj"]
                    t_obj = [strip_objectomeFileName(fn) for fn in ss[1:]]
                    d_obj = [t_ for t_ in t_obj if t_ != s_obj][0]
                    resp = strip_objectomeFileName(r)

                if (s_id in img_oi) & (d_obj in obj_oi):
                    rec_curr = (s_obj,) + (d_obj,) + (resp,) + (s_id,) + (subj["WorkerID"],) + (subj["AssignmentID"],)
                    trial_records.append(rec_curr)

        self.trials = tb.tabarray(records=trial_records, names=self.trial_kw_names, formats=self.trial_kw_formats)
        return

def get_monkeyturk_data(dataset="objectome24"):
    if dataset == "objectome24":
        meta_path = "/mindhive/dicarlolab/u/rishir/stimuli/objectome24s100/metadata.pkl"
        data_path = "/mindhive/dicarlolab/u/rishir/monkeyturk/allData.mat"

    meta = pk.load(open(meta_path, "r"))
    datmat = io.loadmat(data_path)
    uobjs = obj.models_combined24

    col_data_seg = {}
    trial_records = []
    subjs = ["Manto", "Zico", "Picasso", "Nano", "Magneto"]
    for sub in subjs:
        x = datmat["allData"][sub][0, 0]
        for xi in range(x.shape[0]):
            s_obj = uobjs[x[xi, 0]]
            d_obj = uobjs[x[xi, 2]]
            resp = uobjs[x[xi, 3]]
            s_id = meta[x[xi, 4] - 1]["id"]
            workid = sub
            assnid = "MonkeyTurk"

            rec_curr = (s_obj,) + (d_obj,) + (resp,) + (s_id,) + (workid,) + (assnid,)
            trial_records.append(rec_curr)

    col_data_seg["all"] = tb.tabarray(records=trial_records, names=KW_NAMES, formats=KW_FORMATS)
    for sub in subjs:
        t = col_data_seg["all"]["WorkerID"] == sub
        col_data_seg[sub] = col_data_seg["all"][t]
    return col_data_seg

    def get(self):
        meta = tb.tabarray(SVfile=os.path.join(RESULTS_ROOT, 'meta_with_margin_test.tsv'))
        
        N = 3
        days = 4
        text = ''
        NP = 200
        NN = 200
        mdp = meta[:NP]
        mdp = mdp[np.random.RandomState(None).permutation(NP)]['filename']
        mdn = meta[-NN:]
        mdn = mdn[np.random.RandomState(None).permutation(NN)]['filename']

        for d in range(days):
            text += '<div class="entry" id="day_header_%d"><div class="entryheader"><h2 class="entrytitle">Day %d</h2>' % (d, d+1)
            text += '<div class="date">posted by <a href="/vistats_blog">Vistats</a> on 2013.06.19</div></div>'
            ctext = "So, here are my Chanel pics of the day :)"
            text += '<p class="chanel_header" id="chanel_header_day_%d">%s</p><br/>' % (d, ctext)
            chaneltext = '<div class="img_div" id="chanel_img_div_day_%d">' % d + ''.join(['<div class="show_img" id="chanel_img_day_%d_img_%d"><img src="%s/%s"></div>' % (d, _i, IMG_ROOT,x.split('/')[-1]) for _i, x in enumerate(mdp[d*N:(d+1)*N])]) + '</div>'
            text += chaneltext
            notchaneltext = '<div class="img_div" id="not_chanel_img_div_day_%d">' % d + ''.join(['<div class="show_img" id="not_chanel_img_day_%d_img_%d"><img src="%s/%s"></div>' % (d, _i, IMG_ROOT, x.split('/')[-1]) for _i, x in enumerate(mdn[d*N:(d+1)*N])]) + '</div>'
            nctext = "<br/>Hey, and of course I also have a life <b>outside</b> of Chanel :)"
            text += '<p class="not_chanel_header" id="not_chanel_header_day_%d">%s</p><br/>' % (d, nctext) + notchaneltext
            text += '</div>'

        html = HTML_TEMPLATE % text
        
        self.write(html)

        self.finish()