Python joblib.load函数代码示例

    def predict(self, img_path):
        img, positions, pix_data, captcha_type = self.read_img(img_path)
        print positions, captcha_type
        if positions is None:
            print('图像切分错误！')
            return None
        x = np.array(self.get_pix_list(pix_data, positions, captcha_type))
        if captcha_type == 'number':
            if self.model is None or os.path.isfile(self.number_model_file):
                self.model = joblib.load(self.number_model_file)
            else:
                raise IOError
        elif self.model is None or os.path.isfile(self.symbol_model_file):
                self.model = joblib.load(self.symbol_model_file)
        else:
            raise IOError
        predict_label = list()
        for i in range(x.shape[0]):
            input = x[i, :]
            predict_y = self.model.predict(input)[0]
            if int(predict_y) >= len(self.number_label_list) or int(predict_y) < 0:
                return "", ""
            if captcha_type == 'number':
                predict_label.append(self.number_label_list[predict_y])
            else:
                predict_label.append(self.symbol_label_list[predict_y])

        return u"".join(predict_label), self.__caculate(predict_label, captcha_type)

def selectFeatures(X,t=0):
    if(t==0):
        selector=joblib.load('selector.pkl')
    else:
        selector=joblib.load('SelectKBest.pkl')
    X_new=selector.transform(X)
    return X_new

 def load_model(self, path):
     self.clf = joblib.load(os.path.join(path, 'model.pkl'))
     with open(os.path.join(path, 'labels.json'), 'r') as fo:
         self.labels = Alphabet.from_dict(json.load(fo))
     with open(os.path.join(path, 'model_info.json'), 'r') as fo:
         self.model_info = json.load(fo)            
     self.features = joblib.load(os.path.join(path, 'featvec.pkl'))

def train_classifier():
    pos_feat_path = positive_features_path
    neg_feat_path = negative_features_path

    model_path = classifier_model_path

    feature_vectors = []
    labels = []

    for feat_path in glob.glob(os.path.join(pos_feat_path, "*.feat")):
        fd = joblib.load(feat_path)
        print len(fd)
        if len(fd):
            fd = fd.astype(numpy.object)
            feature_vectors.append(fd)
            labels.append(1)

    for feat_path in glob.glob(os.path.join(neg_feat_path, "*.feat")):
        fd = joblib.load(feat_path)
        print len(fd)
        if len(fd):
            fd = fd.astype(numpy.object)
            feature_vectors.append(fd)
            labels.append(0)

    classifier = LinearSVC()
    print "Training classifier"
    classifier.fit(feature_vectors, labels)
    print "Classifier successfully trained"
    if not os.path.isdir(os.path.split(model_path)[0]):
        os.makedirs(os.path.split(model_path)[0])
    joblib.dump(classifier, model_path)

def train_pipeline(kind, cut, vectorizer, model_trainer, do_cut=False, do_vectorizer=False, record_num=None):
    print('reading...')
    alltext, accu_label, law_label, time_label = data.read_trainData("./data/data_train.json", record_num)

    if do_cut:
        print('cutting...')
        train_text = cut.cut(alltext)
        joblib.dump(train_text, './data/{}_cut_train.txt'.format(cut.name))

        print('cleaning...')
        cleaner = Cleaner()
        cleaned_train_text = cleaner.clean(train_text)
        joblib.dump(cleaned_train_text, './data/{}_cut_train_cleaned.txt'.format(cut.name))
    else:
        print('load existing cut file {}...'.format('./data/{}_cut_train_cleaned.txt'.format(cut.name)))
        cleaned_train_text = joblib.load('./data/{}_cut_train_cleaned.txt'.format(cut.name))

    vectorizer_name = '{}_{}'.format(cut.name, vectorizer.name)
    if do_vectorizer:
        print('{} training...'.format(vectorizer_name))
        vectorizer = vectorizer.train(cleaned_train_text)
        joblib.dump(vectorizer,
                    './model/{}/predictor/model/{}_vectorizer.model'.format(model_trainer.name, vectorizer_name))
        print('{} vectorizing...'.format(vectorizer))
        vec = vectorizer.transform(cleaned_train_text)
        joblib.dump(vec, './data/vec_{}.txt'.format(vectorizer_name))
    else:
        print('load existing vec file {}...'.format('./data/vec_{}.txt'.format(vectorizer_name)))
        vec = joblib.load('./data/vec_{}.txt'.format(vectorizer_name))

    print('{} training...'.format(kind))
    model = model_trainer.train(vec, accu_label)
    joblib.dump(model, './model/{}/predictor/model/{}_{}.model'.format(model_trainer.name, vectorizer_name, kind))

def event2semsim(event):
    import os
    from sklearn.externals import joblib
    if isinstance(event, str):
        etype = event
    else:
        etype = event.type
    if etype == "accident":
        return joblib.load(os.path.join(
            os.getenv("TREC_DATA"),
            "semsim", "accidents.norm-stem.lam20.000.pkl"))
    elif etype== "earthquake" or etype == "storm" or etype == "impact event":
        return joblib.load(
            os.path.join(
                os.getenv("TREC_DATA"), 
                "semsim", "natural-disasters.norm-stem.lam20.000.pkl"))
    elif etype == "protest" or etype == "riot":
        return joblib.load(
            os.path.join(
                os.getenv("TREC_DATA"), 
                "semsim", "social-unrest.norm-stem.lam1.000.pkl"))
    elif etype == "shooting" or etype == "bombing" or etype == "conflict" or \
            etype == "hostage":
        return joblib.load(os.path.join(
            os.getenv("TREC_DATA"),
            "semsim", "terrorism.norm-stem.lam10.000.pkl"))

def trainModel():

	# 数据预处理
	data_train = joblib.load('data/data_train.pkl')
	label_train = joblib.load('data/label_train.pkl')

	print data_train.shape

	clf = svm.SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.1, degree=0.1, gamma=1.0,
  kernel='rbf', max_iter=-1, probability=False, random_state=None,
  shrinking=True, tol=0.001, verbose=True)

	#clf.set_params(kernel='rbf')

	print clf

	print data_train.shape
	print label_train.shape

	print 'begin training....'
	clf.fit(data_train,label_train)
	print 'finish training....'
	print clf
	joblib.dump(clf, 'model/svm.pkl')
	
	return None

def train_and_single_label(train_filename, test_filename, clf, pickled):
    """ Only return one example ID for each q_id
    """
    if pickled:
        train_data = joblib.load(train_filename)
        test_data = joblib.load(test_filename)
    else:
        train_data = extract_ibm_data(train_filename)
        test_data = extract_ibm_data(test_filename, test_file=True)

    X = train_data["data"]
    y = train_data["target"]
    clf.fit(X, y)

    labels = clf.predict(test_data["data"])
    # now manipulate the results using test_data['q_id'] to filter the labels
    ##NEW CODE:
    used_qids = []
    results = []
    for i in range(len(labels)):
        if labels[i] == "true":
            if not test_data["q_id"][i] in used_qids:
                results.append(test_data["id"][i])
                used_qids.append(test_data["q_id"][i])
    return results

    def getClassifiers(self):
        if not os.path.exists(self.outDir):
            os.mkdir(self.outDir)
        outDir = self.outDir + os.sep + "classPickle"
        if not os.path.exists(outDir):
            os.mkdir(outDir)
        class1Save = outDir + os.sep + "classifier1.pkl"
        class2Save = outDir + os.sep + "classifier2.pkl"
        
        class1Exists = os.path.exists(class1Save)
        class2Exists = os.path.exists(class2Save)

        if not (class1Exists and class2Exists):
            self._setupTempDir()
            self.fitsFiles = [f[:-5] for f in os.listdir(self.fitsFolder) if ".fits" in f]
            self.fitsFilesLoc = [os.path.abspath(self.fitsFolder + os.sep + f) for f in os.listdir(self.fitsFolder) if ".fits" in f]
            
            for f in self.fitsFiles:
                self.mainCatalog[f] = self.getCatalog(self.fitsFolder + os.sep + f + ".fits", ishape=True)
                self.candidateMask[f] = self._getCandidateMask(self.mainCatalog[f], np.loadtxt(self.fitsFolder + os.sep + f + ".txt"))
                self.mainCatalog[f] = append_fields(self.mainCatalog[f], 'WEIGHT', self.candidateMask[f] * 1.0, usemask=False)    
                self.mainCatalog[f] = append_fields(self.mainCatalog[f], 'EXTENDED', self.candidateMask[f], usemask=False)    
                self.mainCatalog[f] = append_fields(self.mainCatalog[f], 'HLR', np.zeros(self.mainCatalog[f].shape), usemask=False)    
                self.mainCatalog[f] = append_fields(self.mainCatalog[f], 'MAG', np.zeros(self.mainCatalog[f].shape), usemask=False)
            self._trainClassifier()
            joblib.dump(self.sc, class1Save) 
            joblib.dump(self.sc2, class2Save) 
        else:
            self.sc = joblib.load(class1Save)
            self.sc2 = joblib.load(class2Save)
            

        #self._testClassifier(catalog, candidateMask)
        #self._cleanTempDir()
        self._debug("Classifier generated. Now you can invoke .clasify(catalog)")

def loadModule(mode):
    global movieReviewer
    try:
        movieReviewer = joblib.load("./SVM/movieReviewer%s.svm" % mode)
    except:
        import SVMTrain
        movieReviewer = joblib.load("./SVM/movieReviewer%s.svm" % mode)

    def __init__(self):
        if ("model.pkl" in os.listdir()) and ("enc.pkl" in os.listdir()):
            self.model = joblib.load("model.pkl")
            self.enc = joblib.load("enc.pkl")

        else:
            self.refit_from_scratch()

def load_models(path="models",models={}):
    x = os.listdir(path)
    models = models
    for i in x:
        try:
            if not i.startswith('.') and not i.startswith('_') and os.path.isdir(os.path.join(path, i)):
                way = os.path.join(path, i)
                clf = glob.glob(os.path.join(way,"clf_*.pkl"))
                vec = glob.glob(os.path.join(way,"vectorizer_*.pkl"))
                print(". %s"%(way))
                if len(clf)!=1 or len(vec)!=1:
                    print("└── No model found in '%s'. Skipped."%(i))
                    continue
                t0=time()
                sys.stdout.flush()
                print("├── Loading classifier '%s'..."%(i))
                sys.stdout.flush()
                if "clf_%s"%(i) not in models:
                    models["clf_%s"%(i)] = joblib.load(clf[0])
                    print("├── Done. [%.02fs]"%(time()-t0))
                    sys.stdout.flush()
                t0=time()
                print("├── Loading vectorizer '%s'..."%(i))
                sys.stdout.flush()
                if "vectorizer_%s"%(i) not in models:
                    models["vectorizer_%s"%(i)] = joblib.load(vec[0])
                    print("└── Done. [%.02fs]"%(time()-t0))
                    sys.stdout.flush()
                t0=time()
        except:
            print(">> Error on '%s', skipped."%(i))
    return models

def roc_precision_final(db, fac=1):
    if (os.path.exists(MAT_PATH) == False):
        os.mkdir(MAT_PATH)
        
    random_state = check_random_state(0)
    
    print("Loading {}...".format(db))
    clf = joblib.load("clfs/" + db)
        
    classes = clf.classes_
    
    print("Loading test set...")
    loaded = joblib.load("testSet/" + db)
    y_true = loaded[:, -1]

    
    print("Predict proba...")
    y_score = clf.predict_proba(loaded[:, 0:-1])
    loaded = 0
    clf = 0
    y_score = y_score[:, classes == 1] * fac
    
    print("ROC...")
    if (fac != 1):
        db = db + str(fac)
    fpr, tpr, thresholds = roc_curve(y_true, y_score)
    sio.savemat(MAT_PATH + 'final.roc.' + db + '.mat', {'fpr':fpr, 'tpr':tpr, 'thresholds':thresholds})
    
    print("Precision/Recall...")
    precision, recall, thresholds = precision_recall_curve(y_true, y_score)
    sio.savemat(MAT_PATH + 'final.precall.' + db + '.mat', {'precision':precision, 'recall':recall, 'thresholds':thresholds})

    def _train(self, train_data, resources):
        sample_length = len(train_data)
        dict_status_path = os.path.join(root_dic,
                                        'dict_vectorizer_{}.status'.
                                        format(sample_length))
        if os.path.isfile(dict_status_path):
            dictVectorizer = joblib.load(dict_status_path)
        else:
            dictVectorizer = DictVectorizer()
            dictVectorizer.fit(train_data[self.features].
                               fillna(0).
                               to_dict('record'))
            joblib.dump(dictVectorizer, dict_status_path)

        tfidf_status_path = os.path.join(root_dic,
                                         'tfidf_vectorizer_{}.status'.
                                         format(sample_length))
        if os.path.isfile(tfidf_status_path):
            tfidf = joblib.load(tfidf_status_path)
        else:
            tfidf = TfidfVectorizer(min_df=40, max_features=300)
            tfidf.fit(train_data.essay)
            joblib.dump(tfidf, tfidf_status_path)

        resources['dictVectorizer'] = dictVectorizer
        resources['tfidf'] = tfidf
        print 'Head Processing Completed'
        return train_data, resources

    def cheapskateItems(df):
        nonlocal state
        print("Making: cheapskateItems")
        if state == 1 and os.path.exists('pickleFiles/voucherToArticle.pkl'):
            voucherDic = joblib.load('pickleFiles/voucherToArticle.pkl')
        elif state == 0 and os.path.exists('pickleFiles/voucherToArticle_test.pkl'):
            voucherDic = joblib.load('pickleFiles/voucherToArticle_test.pkl')
        else:
            voucherDic = {}
            vouchers = df.groupby('voucherID')
            for idx,voucher in vouchers:
                if idx not in voucherDic:
                    voucherDic[idx] = Counter(voucher['articleID']).most_common()[0][0]
            if state == 1:
                joblib.dump(voucherDic,'pickleFiles/voucherToArticle.pkl')
            else:
                joblib.dump(voucherDic,'pickleFiles/voucherToArticle_test.pkl')

        articleSet = set(voucherDic.values())
        cheapArticle = pd.Series(name='cheapArticle',index=df.index)
        for i in df.index:
            article = df['articleID'][i]
            isCheap = 1 if article in articleSet else 0
            cheapArticle.set_value(i,isCheap)
        df['cheapArticle'] = cheapArticle
        return df