Python Grocery.load方法代碼示例

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
def tGrocery():
    outFile = open('testResult.tmp', 'w')
    [trainingSet, benchmark] = pickle.load(open('SampleSeg.pk'))
    testingSet = []
    correctLabel = []
    for i in xrange(len(benchmark)):
        print '%d out of %d' % (i, len(benchmark))
        testingSet.append(benchmark[i][1])
        correctLabel.append(benchmark[i][0]) 
    grocery = Grocery('test')
    grocery.train(trainingSet)
    grocery.save()
    # load
    new_grocery = Grocery('test')
    new_grocery.load()
    Prediction = []
    for i in xrange(len(testingSet)):
        print '%d out of %d' % (i, len(testingSet))
        prediction = new_grocery.predict(testingSet[i])
        Prediction.append(prediction)
        temp = correctLabel[i] + '<-->' + prediction + '  /x01' + testingSet[i] + '\n'
        outFile.write(temp)
    correct = 0
    for i in xrange(len(Prediction)):
        print Prediction[i], correctLabel[i],
        if Prediction[i] == correctLabel[i]:
            correct += 1
            print 'Correct'
        else:
            print 'False'
    print 'Correct Count:', correct
    print 'Accuracy: %f' % (1.0 * correct / len(Prediction))

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
	def __init__(self, keyword):
		print '進行新聞分類'
		(db, cursor) = connectdb()
		cursor.execute("update task set status=1 where keyword=%s", [keyword])
		cursor.execute("select id, title from news where keyword=%s",[keyword])
		news = cursor.fetchall()
		new_grocery = Grocery('static/paris')
		new_grocery.load()

		for item in news:
			tag = new_grocery.predict(item['title'])
			if tag == '新聞背景':
				tag = 1
			elif tag == '事實陳述':
				tag = 2
			elif tag == '事件演化':
				tag = 3 
			elif tag == '各方態度':
				tag = 4
			elif tag == '直接關聯':
				tag = 6
			elif tag == '暫無關聯':
				tag = 7
			cursor.execute("update news set tag=%s where id=%s", [tag, item['id']])
		closedb(db, cursor)
		return

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
class GroceryModel(object):
    def __init__(self):
        self.grocery = Grocery('TextClassify')
    
    def train(self,train_file):
        f = open(train_file,'r')
        line = f.readline().decode('utf8')
        dataset = []
        while line:
            tmp = line.split('\t')
            dataset.append((tmp[0],''.join(tmp[1:])))
            line = f.readline().decode('utf8')
        f.close()
        self.grocery.train(dataset)
        self.grocery.save()
    
    def load_model(self):
        self.grocery.load()
    
    def test(self,test_src):
        self.load_model()
        f = open(test_src,'r')
        line = f.readline().decode('utf8')
        dataset = []
        while line:
            tmp = line.split('\t')
            dataset.append((tmp[0],''.join(tmp[1:])))
            line = f.readline().decode('utf8')
        f.close()
        result = self.grocery.test(dataset)
        print result
    
    def predict(self,text):
        print self.grocery.predict(text)

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
def test_grocery():
    grocery = Grocery('model_redian')
    grocery.train('trdata_4.txt')
    grocery.save()
    new_grocery = Grocery('model_redian')
    new_grocery.load()
    test_result = new_grocery.test('tedata_4.txt')
    print test_result.accuracy_labels
    print test_result.recall_labels
    test_result.show_result()

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
	def GET(self,name):
		#i = web.input(name=None)	
		#url = "http://"+name
		#html = urllib2.urlopen(url).read()
		#soup = BeautifulSoup(html)
		#title =  soup.html.head.title.contents.pop().encode('utf-8')
		title = name.encode('utf-8')
		new_grocery = Grocery('sample')
		new_grocery.load()
		return new_grocery.predict(title)

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
 def test_main(self):
     grocery = Grocery(self.grocery_name)
     grocery.train(self.train_src)
     grocery.save()
     new_grocery = Grocery('test')
     new_grocery.load()
     assert grocery.get_load_status()
     assert grocery.predict('考生必讀：新托福寫作考試評分標準') == 'education'
     # cleanup
     if self.grocery_name and os.path.exists(self.grocery_name):
         shutil.rmtree(self.grocery_name)

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
class MyGrocery(object):
  def __init__(self, name):
    super(MyGrocery, self).__init__()
    self.grocery = Grocery(name)
    self.loaded = False
    self.correct = 1.0

  def train(self, src):
    lines = []
    for line in csv.reader(open(src)):
      label, s = line[0],line[1]
      text = s.decode('utf8')
      lines.append((label, text))
    self.grocery.train(lines)

  def save_model(self):
    self.grocery.save()

  def train_and_save(self, src):
    self.train(src)
    self.save_model()

  def load_model(self):
    if not self.loaded:
      self.grocery.load()
      self.loaded = True

  def predict(self, text):
    self.load_model()
    return self.grocery.predict(text)

  def test(self, src):
    self.load_model()
    total, wrong_num = 0.0, 0.0
    for line in csv.reader(open(src)):
      total += 1
      if line[0] != self.predict(line[1]):
        wrong_num += 1

    print "load test file from " + src
    correct = (total - wrong_num ) / total
    self.correct = correct
    print "total: %d , wrong_num: %d, success percentage: %f" %(total, wrong_num, correct)
    result = dict(type="test", total=total, wrong_num=wrong_num, correct=correct)
    return json.dumps(result)

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
def predict_corpus(input_file,output_csv):
    import csv
    csvfile = file(output_csv, 'wb')
    writer = csv.writer(csvfile)
    corpus = []
    f = xlrd.open_workbook(input_file)
    table = f.sheet_by_name('Sheet1')
    nrows = table.nrows  # 讀取行數
    for rownum in range(0, nrows):
        row = table.row_values(rownum)
        row[2].strip()
        corpus.append(row[2])
    corpus_grocery = Grocery(project_name)
    corpus_grocery.load()
    output = []
    for sentence in corpus:
        predict = corpus_grocery.predict(sentence)
        output.append((sentence,predict))
    writer.writerows(output)
    print('Done!')
    csvfile.close()

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
class jdParser(object):

    def __init__(self):
        self.clf = Grocery("./jdclf")
        self.clf.load()
        self.LINE_SPLIT = re.compile(u"[；。;\n]")



    def get_demand_and_duty(self,jdstr):
        linelist = [ line.strip() for line in self.LINE_SPLIT.split(jdstr) if len(line.strip()>4) ]

        result = {}
        demand = []
        duty = []
        for line in linelist:
            pred = str(self.clf.predict(line))
            if pred =="demand":
                demand.append(line)
            elif pred == "duty":
                duty.append(line)

        result['demand'] = '\n'.join(demand)
        result['duty'] = '\n'.join(duty)

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.path.append('../../')
from config import *

from tgrocery import Grocery
STOP_WORDS_FILE = 'stopwords.txt'
USER_DICT_FILE = 'user_dict.txt'

model_fintext = Grocery('model_fintext')
model_fintext.load()
sys.path.append('../')
from get_es import *
es = Elasticsearch([{'host':ES_HOST,'port':ES_PORT}])

def search(index_name):
    es_search_options = set_search_optional()
    es_result = get_search_result(es_search_options,index=index_name)
    # final_result = get_result_list(es_result)
    # return final_result
    return es_result


def get_result_list(es_result):
    final_result = []
    for item in es_result:
        final_result.append(item['_source'])
    return final_result

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
##########################################
# init
model_choose = "svm"  # svm, lda, rnn
grocery_name = "./SVM_models/svm_for_news"
corpus_path = "./Corpus/NewsClassCorpus/"
file_path = "./"
file_name = "post.txt"

t_text = delete_stop_words(codecs.open(file_path + file_name, encoding="UTF-8").read())

###########################################
# 調用 SVM 模型分類
if model_choose == "svm":
    tic = time.time()
    grocery = Grocery(grocery_name)
    grocery.load()
    t_pre_result = grocery.predict(delete_stop_words(t_text))
    toc = time.time()

    t_label = t_pre_result.predicted_y
    print("Sentiment: ", t_label)
    print("How much: ", t_pre_result.dec_values[t_label])
    print("Elapsed time of predict is: %s s" % (toc - tic))
elif model_choose == "lda":
    pass
elif model_choose == "rnn":
    pass
else:
    print("")

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
class JdCRF(object):
    def __init__(self):
        self.data = []
        self.clf = Grocery("jdclf")
        self.clf.load()
        
        self.SEX = re.compile(u"性別不限|性別|男|女")
        self.AGE = re.compile(u"\d+周?歲|年齡")
        self.DEGREE = re.compile(u"(全日製)?(初中|高中|中專|大專|專科|大學專科|中職|本科|大學本科|碩士|研究生|博士|博士後)(.?以上)?")
        self.MAJOR = re.compile(u"\S+(相關專業|專業優先|及其.專業|[類等]專業[優先]?)")
        self.EXP = re.compile(u"工作經驗：|工作經[曆驗]|工作年限|年.{0,4}經[曆驗]|經[曆驗].{1,6}年")
        self.PUB_TIME = re.compile(u"(\d+)(天前發布)")
        
        self.INCNAME = re.compile(u"\S+(有限公司|酒店|銀行|集團|研究中心|研究所|學校|旅行社|分?公司|研發中心|技術部|事.部|招聘)") 
        self.NOT_INC = re.compile(u"職位|描述|收藏|推薦|地址|郵箱|主頁|介紹|歡迎|加入|要求|簡介|險一金|獎金|包吃住|社區|廠房|人員|職責") 
        self.INCTAG = re.compile(u"大公司|五百強|全球500強|小公司|成長型公司|創業公司|私有經濟|集體經濟|集團|外企|已上市|穩定性高|平均年齡\d歲|妹紙多|學曆高|福利待遇好|晉升機會大|民營公司|民營企業\
                                 |互聯網|創業型|國企|央企")

        self.JOBNAME = re.compile(u'\S*(研發工程師|工程師|經理|助理|顧問|前台|秘書|主管|研究員|實習生|操作員|專員|教學人員|技術人員|管理員|業務員|公關|程序員|教師|老師|培訓生|\
                                  文員|研究員|策劃|主任|總監|設計師|分析師|架構師|攝影師|編輯|BD|遊戲UI|Android(開發)?|PHP(開發)?|Python(開發)?|.?(急招|急聘|初級|中級|高級|方向).?[\s）】\)])|\
                                  |行政人事|網店設計|客服|會計|電話銷售|外貿跟單|web前端|遊戲UI|後.開發|產品運營|商業數據分析')

        self.START_DEMAND = re.compile(u"(崗位要求|應聘條件|任職要求|崗位資格|任職資格|崗位條件|工作要求|任職條件|人員條件|職位.求|職位條件|職位描述|崗位資格|職位資格|具備條件)[:：\s]\
                                       |如果你.{0,10}[:：\s]|我們希望你.{0,12}[:：\s]|(要求|條件)[:：\s]|你需要?具備什麽.+[？\?:：\s]|任職資格[:：\s]")

        self.DEMAND = re.compile(u"熟悉|熟練|具有|善於|懂得|掌握|具備|能夠|優先|不少於|不超過|至少|團隊.作|良好的|工作經驗|開發經驗|實習經曆|能力強|富有|以上學曆|經驗|喜歡|\
                                 較強的.{2,8}能力|相關專業|相關學曆|者優先|精通|了解|及以上|技術全麵|.強的責任心|[能有]獨立|英文流利")

        self.DUTY = re.compile(u"跟進|協助|負責|配合|其他工作|領導交辦的|對.+提供|審核|參與|提出|跟蹤|報告|為.+提出|日常.+工作|指導|跟進|拓展|運營|用戶|客戶|協調|擬寫|通過|協同\
                               |完成|溝通|需求|秘書.{2,5}翻譯")

        self.START_DUTY = re.compile(u"(崗位職責|崗位描述|職位描述|職責描述|任職描述|職位職責|工作職責|工作職能|職位職能|工作內容|實習內容|職位內容)[:：\s]|做這樣的事[:：\s]|職責.{0,5}[:：\s]")

        self.PAY = re.compile(u"薪酬|待遇|月薪|薪資|年薪|底薪|\d+k|\d+萬|\d+元|工資|報酬|薪水|福利")

        self.BENEFIT = re.compile(u"周休|補助|補貼|假日|餐補|提成|交通補助|食宿|加班工資|期權|年假|領導|扁平化|管理|氛圍|空間|休假|月假|帶薪|全休|晉升|培訓|舒適的|旅遊|獎勵|過節費|五險一金|獎金|\
                                  |彈性工作|氛圍|成長空間|實訓|培訓|高薪|前景|旅遊|活動|分紅")
        
    


    def gen_data(self,fname='./data/lagou_train.txt'):
        fw = codecs.open('./data/jd_train_crf.txt','wb','utf-8')
        cnt = 1
        for line in codecs.open(fname,'rb','utf-8'):
            if line.startswith(u"====="):
                fw.write(line)
                continue

            cnt +=1
            if len(line.strip())>1:
                    pred = self.clf.predict(line)
                    newline = pred+'\t\t'+line.strip()+'\t\t'+str(len(line))+"\n"
                    fw.write(newline)
        print cnt
        print 'done'


    def load_data(self,fname="./data/jd_train_crf.txt"):
        data = []
        tmp = []
        for line in codecs.open(fname,'rb','utf-8'):
            if line.startswith(u"===="):
                data.append(tmp)
                tmp = []
                continue
            else:
                tag_data = line.strip().split('\t\t')
                if len(tag_data)==3:
                    tmp.append(tuple(tag_data))
                else:
                    print '\t  '.join(tag_data)

        
        n = len(data)/2
        print 'train data',n
        print 'test data',len(data)-n
        return data[n:],data[:n]
    

    def word2features(self,sent,i):
        word = sent[i][0]
        postag = sent[i][1]

        features = [
            'bias',
            'word.lower=' + word.lower(),
            'word[:2]=' +word[:2],
            'word.isdigit=%s'%word.isdigit(),
            'postag='+postag,
            'demand=%s'% '1' if self.DEMAND.search(word) else '0',
            'start_demand=%s'% '1' if self.START_DEMAND.search(word) else '0',
            'start_duty=%s'% '1' if self.START_DUTY.search(word) else '0',
            'duty=%s'% '1' if self.DUTY.search(word) else '0',
            'jobname=%s'% '1' if self.JOBNAME.search(word) else '0',
            'incname=%s'% '1' if self.INCNAME.search(word) else '0',
            'benefit = %s'% '1' if self.BENEFIT.search(word) else '0',
            'pred=%s' % self.clf.predict(word)
        ]

#.........這裏部分代碼省略.........

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import MySQLdb
from tgrocery import Grocery
import sys
reload(sys)
sys.setdefaultencoding('utf8')

grocery = Grocery('sample')
dict_list = list()

conn = MySQLdb.connect(host = 'localhost', db = 'newsdata', user = 'root', passwd = 'root', charset = 'utf8', use_unicode = False)
cur = conn.cursor()
cur.execute('select com_new_type_id, com_new_name from tbl_new where com_new_type_id is not null')
for row in cur.fetchall():
    dict_list.append(row)


grocery.train(dict_list)
grocery.save()

news_grocery = Grocery('sample')
news_grocery.load()
while True:
    result = news_grocery.predict(raw_input('please input title:' ))
    print result

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
class JdParserTop(object):


    def __init__(self):
        self.CLEAN_TEXT = re.compile(u"[^\u4e00-\u9fa5\w\d；:：;,。、\.，。！[email protected]（）\r\n\(\)\-\+ －　]")
        
        self.clf = Grocery(base_dir+"/jdclf")
        self.clf.load()
        
        self.SPLIT_LINE = re.compile(u"[\r\n；:：。！？;]|[　\s \xa0\u724b]{4,}")
        self.CLEAN_LINE = re.compile(u"^[\u2022（【\[\s\t\r\n\(\- 　]?[\da-z１２３４５７８９]{1,2}[\.,。、，：:）】\]\)\s]|^[！＠＃￥％……＆×（）\(\)｛｝：“｜、－\-，。：:\.]|^[一二三四五六七八九１２３４５６７８９\d]{0,2}[\.、\s:：　]|[，；。、\s　\.]$|^[\s　\u2022 \uff0d \u25cf]")
        self.CLEAN_JOBNAME = re.compile(u"急聘|誠聘|高薪|包[食住宿餐]|.險一金|待遇|^急?招|職位編號\s?[\s\d:：]")

        self.PAY = re.compile("(\d{3,}\-)?\d{3,}元")
        self.SEX = re.compile(u"性別|男|女")
        self.AGE = re.compile(u"\d+周?歲|年齡")
        self.JOB_TAG = re.compile(u"全職|實習")
        self.DEGREE = re.compile(u"小學|初中|高中|職技|本科|研究生|碩士|博士|教授|專科|大專|中專|無要求|不限|無限")

        self.START_DEMAND = re.compile(u"(任職資格|崗位要求|工作要求|任職條件|任職要求|職位要求)[：:\s】\n　]?")
        self.START_DUTY = re.compile(u"(工作內容|崗位職責|工作職責|職位描述|工作描述|職位介紹|職位職責|崗位描述)[:：\s 】\n　]")
        self.START_BENEFIT = re.compile(u"(福利待遇|待遇|福利)[:：\s\n】]")
        
        self.INC_URL = re.compile(u"(主頁|網站|網址|官網).{0,3}[\w\d_/\.:\-]+")
        self.DEMAND = re.compile(u"精通|熟悉|熟練|有.+經驗")
        self.DUTY = re.compile(u"負責|促成|為客戶|安排的其.工作")
        self.BENEFIT = re.compile(u".險一金|福利|晉身|休假|帶薪|補助|補貼")
        self.CERT = re.compile(u"(\S{2,8}證書|CET-\d|普通話|英語|口語|.語|日文|雅思|托福|托業)(至少)?(通過)?[\d一二三四五六七八九]級[及或]?(以上)?|(英語)?CET-\d級?(以上)?|\
                                 醫學.{0,3}證|會計.{0,3}證|律師.{0,3}證|有.{1,8}證書")


        self.degreedic = set([line.strip() for line in codecs.open(base_dir+'/data/degrees.txt','rb','utf-8')])
        self.majordic = set([line.strip() for line in codecs.open(base_dir+'/data/majordic.txt','rb','utf-8')])
        self.skilldic = set([line.strip() for line in codecs.open(base_dir+'/data/skills.txt','rb','utf-8')])
        self.jobdic = set([line.strip() for line in codecs.open(base_dir+'/data/jobnames.txt','rb','utf-8')])

        jieba.load_userdict(base_dir+'/data/majordic.txt')
        jieba.load_userdict(base_dir+'/data/skills.txt')
        jieba.load_userdict(base_dir+'/data/firm.txt')
        jieba.load_userdict(base_dir+'/data/degrees.txt')
        jieba.load_userdict(base_dir+'/data/benefits.txt')

    
    def clean_line(self,line):
        """
        清除一個句子首尾的標點符號
        """
        line = self.CLEAN_LINE.sub("",line).strip()
        line = re.sub("\s+|^/d+[；’、，／。\.]","",line)
        return line


    def clean_cnNum(self,line):
        """
        經驗年限提取時，中文一二三等轉為123
        """
        line = unicode(line)
        a = [u"一",u"二",u"三",u"四",u"五",u"六",u"七",u"八",u"九",u"十",u"兩"]
        b = range(1,11)+[2]
        table = dict((ord(aa),bb) for aa,bb in zip(a,b))
        return line.translate(table)




    def line2vec(self,line):
        """
        句子轉換為向量
        """
        vec = np.zeros(50)
        for word in jieba.cut(line):
            if word in self.w2v.vocab:
                vec += self.w2v[word]

        return vec
    
    
    def clean_jobname(self,jobname):
        """
        職位名清洗
        """
        if jobname.lower() in self.jobdic:
            return jobname.lower()
        else:
           res = [(lcs_len(jobname,job),job) for job in self.jobdic]
           res.sort()
           return res[-1][1]

# 需要導入模塊: from tgrocery import Grocery [as 別名]
# 或者: from tgrocery.Grocery import load [as 別名]
		tdic['id'].append(_id)
		tdic['type'].append(_type)
		tdic['contents'].append(contents)
	i +=1
	
#train = pd.read_csv( train_file, header = 0, delimiter = "\t", quoting = 3 )
#test = pd.read_csv( test_file, header = 1, delimiter = "\t", quoting = 3 )
train = DataFrame(dic)
test = DataFrame(tdic)
#
#classfynews_instance 是模型保存路徑
grocery = Grocery('classfynews_instance')

train_in = [train['contents'],train['type']]
grocery.train(train_in)
print grocery.get_load_status()
#grocery.save()

copy_grocery = Grocery('classfynews_instance')
copy_grocery.load()
#copy_grocery = grocery
test_in = [test['contents'],test['type']]
#輸入類似 ['我是中國人','台北*****']
#輸出 [11,12]
test_result = copy_grocery.predict(test['contents'])
print test_result.predicted_y
#test_result = copy_grocery.test(test_in)
#print test_result.show_result()