Python IOtools.todisc_txt方法代码示例

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def img_ref_captions():
    rpath = "/home/dicle/Dicle/Tez/geziyakurdiproject/corpus2/ldatests22Temmuz/wordletest/words/temp/"
    metadf = pd.read_csv(rpath+"/metadocs.csv", index_col=None, sep="\t")
    
    header = '\\begin{figure}[ht] \n \
\subfigure[frequency weighted word cloud]{ \n \
 \includegraphics[width=3.5in]{pics_docs/freq_'
 
    middle1 = '.png}} \n \
\quad \
\subfigure[tfidf weighted word cloud]{ \n \
 \includegraphics[width=3in]{pics_docs/tfidf_'
 
    middle2 = ".png}} \n \
                \caption{ " 

    end = "\end{figure}"
           
    outltx = ""
    numofdocs, fields = metadf.shape
    for i in range(numofdocs):
        filename = metadf.loc[i, "filename"]
        author = metadf.loc[i, "Author"]
        title = metadf.loc[i, "Title"]
        link = metadf.loc[i, "Link"]
        date = metadf.loc[i, "Date"]
        resource = metadf.loc[i, "Resource"]
        
        caps_link = "\href{" + link + "}"
        caps_a = "{\\textit{" + author + "}, " + title + ", \\textit{" + date + "} - @" + resource + "} }\n"
        figtxt = header + filename + middle1 + filename + middle2 + caps_link + caps_a + end
        
        outltx = outltx + figtxt + "\n\n"
    
    IOtools.todisc_txt(outltx, rpath+"docswordle_figLaTeX.txt")

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def combine_features(self, combmatrix):
     ncombs, nrows = combmatrix.shape
     
     for i,row in enumerate(combmatrix):
         filename = "comb"+str(i)+"_F"
         featuredflist = []
         for j,featno in enumerate(row):
             groupname = sorted(self.featuremap.keys())[j]
             filename += "_"+str(j)+"-"+str(featno)   # filename = combNO_F_GROUPNO-FEATNO
                             
             extractorinstance = self.featuremap[groupname][featno]
             featurematrixpath = extractorinstance.getfeaturematrixpath
             featurematrix = IOtools.readcsv(featurematrixpath, keepindex=True)
             featuredflist.append(featurematrix)
         
         print filename
         print utils.decode_combcode(filename, self.featuremap)
         datamatrix = pd.concat(featuredflist, axis=1) #, verify_integrity=True) # CLOSED DUE TO THE OVERLAPPING WORDS IN ABS AND SUBJ LISTS
         #datamatrix['index'] = datamatrix.index
         #datamatrix = datamatrix.drop_duplicates(cols='index')
         #del datamatrix['index']
         
         # replace nan and inf cells !! no. work on matrix, not df. better do this change on learning
         #datamatrix[np.isnan(datamatrix)] = 0
         #datamatrix[np.isinf(datamatrix)] = 0
         
         datamatrixpath = self.combinedfeaturesfolder + os.sep + filename + ".csv"
         IOtools.tocsv(datamatrix, datamatrixpath, keepindex=True)
         
         # record comb name decoding
         decodednamesfolder = IOtools.ensure_dir(os.path.join(self.datasetrootpath, metacorpus.decodedcombnamesfoldername))
         decodedname = utils.tostr_decoded_combcode(filename, self.featuremap)
         IOtools.todisc_txt(decodedname, os.path.join(decodednamesfolder, filename+".txt"))

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def csv2latextable_algorithm(inpath, outpath, filename, metricname):
    
    header = "\\begin{table}[h] \n \
\\begin{center} \n \
\\begin{tabular}{|p{9cm}|p{2cm}|p{2cm}|p{2cm}|} \n \
\\hline  \\bf algorithm \& parameters & \\bf mean "+ metricname +" & \\bf minimum "+ metricname +" & \\bf maximum "+ metricname +"   \\\ \\hline"
    
    footer = "\\end{tabular} \n \
\\end{center} \n \
\\caption{\\label{alg-"+metricname[:4]+"-stats} Mean, maximum and minimum "+metricname+" results for 27 learning models } \n \
\\end{table}"
    
    ip1 = os.path.join(inpath, filename+".csv")
    df = IOtools.readcsv(ip1, keepindex=True)
    nrows, ncols = df.shape
    rowids = df.index.values.tolist()
    
    out = header+"\n"
    for rowid in rowids:
        featset = rowid[4:]
        featset = "\\verb|"+featset+"|"
        
        out += featset + " & "
        #np.round(a, decimals, out)
        mean = df.loc[rowid, "mean"]
        min = df.loc[rowid, "min"]
        max = df.loc[rowid, "max"]
        stats = map(lambda x : str(round(x, 5)), [mean, min, max])
        statsstr = " & ".join(stats)
        out += statsstr + " \\\ \hline " + "\n"
    
    out += footer
    IOtools.todisc_txt(out, os.path.join(outpath, filename+".txt"))

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def reportresults(self, ytrue, ypred, experimentname):
     
     '''
     precision, recall, f1score, _ = metrics.precision_recall_fscore_support(ytrue, ypred)     
     print precision, recall, f1score
     '''
     #print ytrue
     #print ypred     
        
     precision = metrics.precision_score(ytrue, ypred, pos_label=None, average="macro")
     recall = metrics.recall_score(ytrue, ypred, pos_label=None, average="macro")
     f1score = metrics.f1_score(ytrue, ypred, pos_label=None, average="macro")
     accuracy = metrics.accuracy_score(ytrue, ypred)
     
     scoreline = metaexperimentation.csvsep.join(map(lambda x : str(x), [experimentname, precision, recall, f1score, accuracy]))
     IOtools.todisc_txt("\n"+scoreline, self.scorefilepath, mode="a")
     
     modelscorereportpath = os.path.join(self.experimentrootpath, experimentname+".txt")   
     try:
         scorereportstr = metrics.classification_report(ytrue, ypred, target_names=self.labelnames)
     except:
         scorereportstr = "zero division error\n"
     IOtools.todisc_txt(scorereportstr, modelscorereportpath)
     
     # record instances
     path = modelscorereportpath = os.path.join(self.experimentrootpath, "instances", experimentname+".csv")
     iheader = ["ytrue\t ypred"]
     instances = [str(true)+"\t"+str(pred) for (true, pred) in zip(ytrue, ypred)]
     IOtools.todisc_list(path, iheader+instances)

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def metadata_tabular():
    rpath = "/home/dicle/Dicle/Tez/geziyakurdiproject/corpus2/ldatests22Temmuz/wordletest/words/temp/"
    metadf = pd.read_csv(rpath+"/metadocs.csv", index_col=None, sep="\t")
    
    print metadf.loc[0,"Author"]
    metadf = metadf.sort(["Polarity", "Date", "Author"], ascending=[False, True, True])
    v = metadf.iloc[0,:]
    print v.loc["Author"],v.loc["Resource"]
    
    header = "\\begin{tabular}{l  | c | c | c | c } \n \
kategori & yazar & başlık & tarih & yayın \\\\ \n \
\\hline \\hline \n"

    end = "\\end{tabular}"
    outltx = ""
    numofdocs, fields = metadf.shape
    for i in range(numofdocs):
        row = metadf.iloc[i,:]
        cat = row.loc["Polarity"]
        cat = "\\textbf{"+cat+"}"
        author = row.loc["Author"]
        title = row.loc["Title"]
        link = row.loc["Link"]
        date = row.loc["Date"]
        resource = row.loc["Resource"]
        
        title = "\\href{"+link+"}{"+title+"}"
        date = "\\textit{"+date+"}"
        resource = "@"+resource
        
        s = " & ".join([cat, author, title, date, resource])
        outltx = outltx + s + "\\\\ \n \\hline \n"
    
    outltx = header + outltx + end
    IOtools.todisc_txt(outltx, rpath+"docswordle_tableLaTeX.txt")

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def csv2latextable_featset(inpath, outpath, filename, metricname):
    
    header = "\\begin{table}[h] \n \
\\begin{center} \n \
\\begin{tabular}{|p{5cm}|p{2cm}|p{2cm}|p{2cm}|} \n \
\\hline  \\bf feature-combined dataset name & \\bf mean "+ metricname +" & \\bf minimum "+ metricname +" & \\bf maximum "+ metricname +"   \\\ \\hline"
    
    footer = "\\end{tabular} \n \
\\end{center} \n \
\\caption{\\label{featset-"+metricname[:4]+"-stats} Mean, maximum and minimum "+metricname+" results for 8 feature-measure-combined datasets } \n \
\\end{table}"
    
    ip1 = os.path.join(inpath, filename+".csv")
    df = IOtools.readcsv(ip1, keepindex=True)
    nrows, ncols = df.shape
    rowids = df.index.values.tolist()
    
    out = header+"\n"
    for rowid in rowids:
        featset = rowid.split("**")[0].strip()
        featset = "\\verb|"+featset+"|"
        out += featset + " & "
        #np.round(a, decimals, out)
        mean = df.loc[rowid, "mean"]
        min = df.loc[rowid, "min"]
        max = df.loc[rowid, "max"]
        stats = map(lambda x : str(round(x, 5)), [mean, min, max])
        statsstr = " & ".join(stats)
        out += statsstr + " \\\ \hline " + "\n"
    
    out += footer
    IOtools.todisc_txt(out, os.path.join(outpath, filename+".txt"))

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def tostr_featuremap(self, record=True):
     s = ""
     for j,k in enumerate(sorted(self.featuremap.keys())):
         line = str(j)+". "+k + " : "
         for i,metric in enumerate(self.featuremap[k]):
             line += str(i)+"- "+str(metric)+", "
         s += line + "\n"
     
     if record:
         IOtools.todisc_txt(s, os.path.join(self.datasetrootpath, "featuremap.txt"))
     return s

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def compare_topical_terms(doctopic1, doctopic2, doclist, path):
        
    out = ""
    for doc in doclist:
        out += "\n\n"+doc[:13]+"\n"
        for w1,w2 in zip(doctopic1[doc], doctopic2[doc]):
                l1, v1 = w1
                l2, v2 = w2
                out += l1+" ,"+str(round(v1,4))+"\t"+l2+" ,"+str(round(v2,4))+"\n"
    
    IOtools.todisc_txt(out, path)    

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def classify(self, Xtrain, ytrain, Xtest, ytest, classifiermodel, modelname):
     p = multiprocessing.Process(target=self.classify_hardcore,
                                  kwargs={"Xtrain":Xtrain, "ytrain":ytrain,
                                          "Xtest":Xtest, "ytest":ytest,
                                          "classifiermodel":classifiermodel,
                                          "modelname":modelname})
     p.start()
     p.join(200)   # quit after 10 min.s
     if p.is_alive():
         print "Quit ",modelname
         IOtools.todisc_txt("", self.experimentrootpath+os.sep+"Quit-"+modelname+".txt")
         p.terminate()
         p.join()

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def reportresults(self, ytrue, ypred, experimentname, scorefilepath, labelnames):
     precision = metrics.precision_score(ytrue, ypred)
     recall = metrics.recall_score(ytrue, ypred)
     f1score = metrics.f1_score(ytrue, ypred)
     accuracy = metrics.accuracy_score(ytrue, ypred)
     
     scoreline = metaexperimentation.csvsep.join(map(lambda x : str(x), [experimentname, precision, recall, f1score, accuracy]))
     IOtools.todisc_txt("\n"+scoreline, scorefilepath, mode="a")
     
     selfscorereportpath = os.path.join(self.experimentrootpath, experimentname+".txt")   
     
     scorereportstr = metrics.classification_report(ytrue, ypred, target_names=labelnames)
     IOtools.todisc_txt(scorereportstr, selfscorereportpath)

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def recordCFD(cfd, filename):
    outstr = "\n"
    for cond in cfd.conditions():
        totaloccurrence = cfd[cond].N()
        outstr = outstr + "\n" + str(cond) + " :\n"
        outstr = outstr + "  cumulative occ. " + str(totaloccurrence)+"\n"
        outstr = outstr + "  the most occ. element. " + str(cfd[cond].max())+"\n"
        
        itemstr = "occurrences:\n"
        for item in list(cfd[cond]):
            itemstr += "\t" + str(item) + " : " + str(cfd[cond][item]) + "\n"
        
        outstr = outstr + itemstr   
    outstr = outstr + "# of conds: " + str(len(cfd.conditions()))
    IOtools.todisc_txt(outstr, IOtools.results_rootpath + os.sep + filename + ".txt")

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
 def test2(self, testpoints, testlabels, classlabel_decode, ldac, filename):
     out = ""  
     out += "Prediction"
     out += "\nNumber of test data: " + str(len(testpoints))
     out += "\nPredicted \t Actual \n"
     predictions =  ldac.pred(testpoints)
     errors = 0
     for p,a in zip(predictions, testlabels):
         #print str(p)+" \t "+str(a)
         out += classlabel_decode[p] + " \t " + classlabel_decode[a] + "\n"
         if str(p) != str(a):
             errors += 1
     errorrate = float(errors) / len(testpoints)
     out += "\nError rate: " + str(errorrate)
     
     IOtools.todisc_txt(out, IOtools.results_rootpath+os.sep+filename)

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def crawlandmakexmlcorpus():
    
    for resource in resourcefolders:
        p1 = os.path.join(rawcorpuspath, resource)
        xp1 = IOtools.ensure_dir(os.path.join(xmlcorpuspath, resource))  # replicate the folder hierarchy into the xml folder as well
        categories = IOtools.getfoldernames_of_dir(p1)
        for cat in categories:
            p2 = os.path.join(p1,cat)
            xp2 = IOtools.ensure_dir(os.path.join(xp1, cat))
            txtfiles = IOtools.getfilenames_of_dir(p2, removeextension=True)
            
            for filename in txtfiles:
                txtpath = p2 + os.sep + filename + fromextension
                xmlpath = xp2 + os.sep + filename + toextension
                txtcontent = IOtools.readtxtfile(txtpath)
                xmlcontent = headxml + "\n" + txtcontent + "\n" + footxml
                IOtools.todisc_txt(xmlcontent, xmlpath)

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def test_classifier(testdata, testlabels, predictions, resource_decode, recordfilename):
    out = ""  
    out += "Prediction"
    out += "\nNumber of test data: " + str(len(testdata))
    out += "\nActual \t Predicted\n"
    
    errors = 0
    for p,a in zip(predictions, testlabels):
        #print str(p)+" \t "+str(a)
        out += resource_decode[p] + " \t " + resource_decode[a] + "\n"
        if str(p) != str(a):
            errors += 1
    errorrate = float(errors) / len(testdata)
    out += "\nAccuracy: " + str(1-errorrate)
    out += "\nError rate: " + str(errorrate)
    
    IOtools.todisc_txt(out, IOtools.results_rootpath+os.sep+recordfilename)

# 需要导入模块: from sentimentfinding import IOtools [as 别名]
# 或者: from sentimentfinding.IOtools import todisc_txt [as 别名]
def recordnewsmetadata_crawltxt(corpuspath=metacorpus.rawcorpuspath, resourcefolders=metacorpus.resources, csvfilepath=_metafilepath):
      
    for resource in resourcefolders:
        xp1 = IOtools.ensure_dir(os.path.join(corpuspath, resource))  # replicate the folder hierarchy into the xml folder as well
        categories = IOtools.getfoldernames_of_dir(xp1)
        
        for cat in categories:
            xp2 = IOtools.ensure_dir(os.path.join(xp1, cat))
            filenames = IOtools.getfilenames_of_dir(xp2, removeextension=False)
            
            for filename in filenames:
                filepath = xp2 + os.sep + filename 
                metadataline = getmetadata_fromtxt(filepath)    #metadataline = getmetadata_fromtxt(filepath+".txt") 
                #print csvfilepath               
                IOtools.todisc_txt(metadataline, csvfilepath, mode="a")
        
            print "finished "+resource+"/"+cat