Python Grocery.train方法代码示例

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
def train():
    print 'train start '+'.'*30
    #grocery=Grocery('sample')
    grocery=Grocery('version1.0')
    grocery.train(trainlist)
    grocery.save()
    print 'train end '+'.'*30

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
def tGrocery():
    outFile = open('testResult.tmp', 'w')
    [trainingSet, benchmark] = pickle.load(open('SampleSeg.pk'))
    testingSet = []
    correctLabel = []
    for i in xrange(len(benchmark)):
        print '%d out of %d' % (i, len(benchmark))
        testingSet.append(benchmark[i][1])
        correctLabel.append(benchmark[i][0]) 
    grocery = Grocery('test')
    grocery.train(trainingSet)
    grocery.save()
    # load
    new_grocery = Grocery('test')
    new_grocery.load()
    Prediction = []
    for i in xrange(len(testingSet)):
        print '%d out of %d' % (i, len(testingSet))
        prediction = new_grocery.predict(testingSet[i])
        Prediction.append(prediction)
        temp = correctLabel[i] + '<-->' + prediction + '  /x01' + testingSet[i] + '\n'
        outFile.write(temp)
    correct = 0
    for i in xrange(len(Prediction)):
        print Prediction[i], correctLabel[i],
        if Prediction[i] == correctLabel[i]:
            correct += 1
            print 'Correct'
        else:
            print 'False'
    print 'Correct Count:', correct
    print 'Accuracy: %f' % (1.0 * correct / len(Prediction))

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
class GroceryModel(object):
    def __init__(self):
        self.grocery = Grocery('TextClassify')
    
    def train(self,train_file):
        f = open(train_file,'r')
        line = f.readline().decode('utf8')
        dataset = []
        while line:
            tmp = line.split('\t')
            dataset.append((tmp[0],''.join(tmp[1:])))
            line = f.readline().decode('utf8')
        f.close()
        self.grocery.train(dataset)
        self.grocery.save()
    
    def load_model(self):
        self.grocery.load()
    
    def test(self,test_src):
        self.load_model()
        f = open(test_src,'r')
        line = f.readline().decode('utf8')
        dataset = []
        while line:
            tmp = line.split('\t')
            dataset.append((tmp[0],''.join(tmp[1:])))
            line = f.readline().decode('utf8')
        f.close()
        result = self.grocery.test(dataset)
        print result
    
    def predict(self,text):
        print self.grocery.predict(text)

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
 def __train__model__():
     dataframe = pd.read_excel(Classify.__FILE_PATH__)
     data = dataframe[[u'类型',	u'释义']]
     train_data = [(x[0],x[1]) for x in data.values]
     
     grocery = Grocery('Classify')
     
     grocery.train(train_data)
     grocery.save()
     Classify.__MODEL__ = grocery

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
def test_grocery():
    grocery = Grocery('model_redian')
    grocery.train('trdata_4.txt')
    grocery.save()
    new_grocery = Grocery('model_redian')
    new_grocery.load()
    test_result = new_grocery.test('tedata_4.txt')
    print test_result.accuracy_labels
    print test_result.recall_labels
    test_result.show_result()

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
 def test_main(self):
     grocery = Grocery(self.grocery_name)
     grocery.train(self.train_src)
     grocery.save()
     new_grocery = Grocery('test')
     new_grocery.load()
     assert grocery.get_load_status()
     assert grocery.predict('考生必读：新托福写作考试评分标准') == 'education'
     # cleanup
     if self.grocery_name and os.path.exists(self.grocery_name):
         shutil.rmtree(self.grocery_name)

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
def sentiment_train(gro_name, train_set):
    """

    :param gro_name:
    :param train_set:
    :return:
    """
    gro_ins = Grocery(gro_name)
    # gro_ins.load()
    gro_ins.train(train_set)
    print("Is trained? ", gro_ins.get_load_status())
    gro_ins.save()

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
class MyGrocery(object):
  def __init__(self, name):
    super(MyGrocery, self).__init__()
    self.grocery = Grocery(name)
    self.loaded = False
    self.correct = 1.0

  def train(self, src):
    lines = []
    for line in csv.reader(open(src)):
      label, s = line[0],line[1]
      text = s.decode('utf8')
      lines.append((label, text))
    self.grocery.train(lines)

  def save_model(self):
    self.grocery.save()

  def train_and_save(self, src):
    self.train(src)
    self.save_model()

  def load_model(self):
    if not self.loaded:
      self.grocery.load()
      self.loaded = True

  def predict(self, text):
    self.load_model()
    return self.grocery.predict(text)

  def test(self, src):
    self.load_model()
    total, wrong_num = 0.0, 0.0
    for line in csv.reader(open(src)):
      total += 1
      if line[0] != self.predict(line[1]):
        wrong_num += 1

    print "load test file from " + src
    correct = (total - wrong_num ) / total
    self.correct = correct
    print "total: %d , wrong_num: %d, success percentage: %f" %(total, wrong_num, correct)
    result = dict(type="test", total=total, wrong_num=wrong_num, correct=correct)
    return json.dumps(result)

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
    if(i%5000==0):
        print ("%d "%(i))+'#'*30
    str=line.split(u',')
    count=str.__len__()
    if(count<2):
        print 'error happen'+"#"*30
        continue

    #print count
    #print str
    trainstr=(str[0],str[1])
    trainlist.append(trainstr)
    #print str[1]+u','+str[2]

grocery=Grocery('sample')
grocery.train(trainlist)
grocery.save()
filein.close()


# test ##################################
print 'start test'
TP=0.0
TN=0.0
FP=0.0
FN=0.0

filetest=codecs.open(validateFileName,'r','utf-8')
test_reader=filetest.readlines()

resultlist=[]

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
from tgrocery import Grocery
data_dir = "../data/"
src_fn = data_dir + 'train_set_100.txt'
grocery = Grocery('backout_reason')
grocery.train(src_fn)

tp_cnt = {}
f = open(data_dir + 'type.txt')
for line in f:
	tps = line.split()
	tp_cnt[tps[1]] = 0

f.close()

f = open(data_dir + 'bcmtmoz.merge')
for line in f:
	tp = grocery.predict(line)
	tp_cnt[tp] += 1

print tp_cnt

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import MySQLdb
from tgrocery import Grocery
import sys
reload(sys)
sys.setdefaultencoding('utf8')

grocery = Grocery('sample')
dict_list = list()

conn = MySQLdb.connect(host = 'localhost', db = 'newsdata', user = 'root', passwd = 'root', charset = 'utf8', use_unicode = False)
cur = conn.cursor()
cur.execute('select com_new_type_id, com_new_name from tbl_new where com_new_type_id is not null')
for row in cur.fetchall():
    dict_list.append(row)


grocery.train(dict_list)
grocery.save()

news_grocery = Grocery('sample')
news_grocery.load()
while True:
    result = news_grocery.predict(raw_input('please input title:' ))
    print result

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
def traditionalize(text):
 return opencc.convert(text, config='zhs2zht.ini').encode('utf-8')

if __name__ == '__main__':
	
	if len(sys.argv) == 6:
		# ======= Trainning News Category =======
		# Depends on current news classifier
		ebCawler = EBCawler()
		# Warning!!! - Need web connection
		ebCawler.getCurrentXML()

		# for data in ebCawler.getTranningData():
		# print len(ebCawler.getTranningData())
		grocery = Grocery('sample')
		grocery.train(ebCawler.getTranningData())

		# ======= Simplify All Original Docs =======
		originDocs_Dir = sys.argv[1]
		outputDocs_Dir = sys.argv[2]
		outputWithCateDocs_Dir = sys.argv[3]
		tranningCSV = sys.argv[4]
		queryDir = sys.argv[5]
		originDocs = OriginDocs(originDocs_Dir, outputDocs_Dir)
		# originDocs.simplifyAllDoc();

		# ======= Category Original Docs and save to json file =======
		# for x in os.listdir(outputDocs_Dir):
		# 	if x == ".DS_Store":
		# 		continue
		# 	content = None

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
train_listc=[]
i=0
q=0
for c in mycontent:
    if c:
        k=mysign[q]
        p=[k,c]
        train_listc.append(p)
        q=q+1

for t in mytitle:
    m=mysign[i]
    n=[m,t]
    train_list.append(n)
    i=i+1
grocery.train(train_listc)
grocery.train(train_list)
grocery.save()
new_grocery=Grocery('trydb')
new_grocery.load()
pc=message.getContent1()
pt=message.getTitle1()
g=1
for newscontent in pc:
    if newscontent:
        num=new_grocery.predict(newscontent+pt[g-1])
        message.saveContent(g,num)
    else:
        num=new_grocery.predict(pt[g-1])
        message.saveContent(g,num)
   

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
# coding: utf-8

from tgrocery import Grocery

# pass a tokenizer, must be a python func
custom_grocery = Grocery('custom', custom_tokenize=list)
train_src = [
    ('education', '名师指导托福语法技巧：名词的复数形式'),
    ('education', '中国高考成绩海外认可 是“狼来了”吗？'),
    ('sports', '图文：法网孟菲尔斯苦战进16强 孟菲尔斯怒吼'),
    ('sports', '四川丹棱举行全国长距登山挑战赛 近万人参与')
]
custom_grocery.train(train_src)
print custom_grocery.get_load_status()
print custom_grocery.predict('考生必读：新托福写作考试评分标准')

# 需要导入模块: from tgrocery import Grocery [as 别名]
# 或者: from tgrocery.Grocery import train [as 别名]
	else :
		tdic['id'].append(_id)
		tdic['type'].append(_type)
		tdic['contents'].append(contents)
	i +=1
	
#train = pd.read_csv( train_file, header = 0, delimiter = "\t", quoting = 3 )
#test = pd.read_csv( test_file, header = 1, delimiter = "\t", quoting = 3 )
train = DataFrame(dic)
test = DataFrame(tdic)
#
#classfynews_instance 是模型保存路径
grocery = Grocery('classfynews_instance')

train_in = [train['contents'],train['type']]
grocery.train(train_in)
print grocery.get_load_status()
#grocery.save()

copy_grocery = Grocery('classfynews_instance')
copy_grocery.load()
#copy_grocery = grocery
test_in = [test['contents'],test['type']]
#输入类似 ['我是中国人','台北*****']
#输出 [11,12]
test_result = copy_grocery.predict(test['contents'])
print test_result.predicted_y
#test_result = copy_grocery.test(test_in)
#print test_result.show_result()