Python Word2Vec.load方法代码示例

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def load_last(self, save_dir):
		'''Load last model from dir
		'''
		def extract_number_of_epochs(filename):
			m = re.search('_ne([0-9]+(\.[0-9]+)?)_', filename)
			return float(m.group(1))

		# Get all the models for this RNN
		file = save_dir + self._get_model_filename("*")
		file = np.array(glob.glob(file))

		if len(file) == 0:
			print('No previous model, starting from scratch')
			return 0

		# Find last model and load it
		last_batch = np.amax(np.array(map(extract_number_of_epochs, file)))
		last_model = save_dir + self._get_model_filename(last_batch)
		print('Starting from model ' + last_model)
		self.load(last_model)

		return last_batch 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def generate_block_seqs(self):
        if self.language is 'c':
            from prepare_data import get_blocks as func
        else:
            from utils import get_blocks_v1 as func
        from gensim.models.word2vec import Word2Vec

        word2vec = Word2Vec.load(self.root+self.language+'/train/embedding/node_w2v_' + str(self.size)).wv
        vocab = word2vec.vocab
        max_token = word2vec.syn0.shape[0]

        def tree_to_index(node):
            token = node.token
            result = [vocab[token].index if token in vocab else max_token]
            children = node.children
            for child in children:
                result.append(tree_to_index(child))
            return result

        def trans2seq(r):
            blocks = []
            func(r, blocks)
            tree = []
            for b in blocks:
                btree = tree_to_index(b)
                tree.append(btree)
            return tree
        trees = pd.DataFrame(self.sources, copy=True)
        trees['code'] = trees['code'].apply(trans2seq)
        if 'label' in trees.columns:
            trees.drop('label', axis=1, inplace=True)
        self.blocks = trees

    # merge pairs 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def generate_block_seqs(self,data_path,part):
        from prepare_data import get_blocks as func
        from gensim.models.word2vec import Word2Vec

        word2vec = Word2Vec.load(self.root+'train/embedding/node_w2v_' + str(self.size)).wv
        vocab = word2vec.vocab
        max_token = word2vec.syn0.shape[0]

        def tree_to_index(node):
            token = node.token
            result = [vocab[token].index if token in vocab else max_token]
            children = node.children
            for child in children:
                result.append(tree_to_index(child))
            return result

        def trans2seq(r):
            blocks = []
            func(r, blocks)
            tree = []
            for b in blocks:
                btree = tree_to_index(b)
                tree.append(btree)
            return tree
        trees = pd.read_pickle(data_path)
        trees['code'] = trees['code'].apply(trans2seq)
        trees.to_pickle(self.root+part+'/blocks.pkl')

    # run for processing data to train 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def get_data():
    train_vecs=np.load(storedpaths+'train_vecs.npy')
    y_train=np.load(storedpaths+'y_train.npy')
    test_vecs=np.load(storedpaths+'test_vecs.npy')
    y_test=np.load(storedpaths+'y_test.npy') 
    return train_vecs,y_train,test_vecs,y_test

# 训练svm模型 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def get_predict_vecs(string,n_dim,w2v_model_path):
    '''
    string: 输入的句子
    n_dim: 词向量维度
    w2v_model_path: 預训练词向量的模型路径
    '''
    n_dim = n_dim
    text_w2v = Word2Vec.load(w2v_model_path)
    words=[i for i in jieba.cut(string,cut_all=False)]
    train_vecs = buildWordVector(words, n_dim,text_w2v)

    return train_vecs

# 调用训练模型进行预测 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def svm_predict(string,trainmodelpath):
    words_vecs=get_predict_vecs(string)
    clf=joblib.load(trainmodelpath)
    result=clf.predict(words_vecs)
    
    return result

# Train model 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def load_pretrained_word_embeddings(self, embedding_path, kernel='kv'):
        trained_embeddings = OrderedDict()
        if kernel == 'gensim':
            from gensim.models.word2vec import Word2Vec

            w2v_model = Word2Vec.load(embedding_path)
            word_dict = w2v_model.wv.vocab
            for token in word_dict:
                if token not in self.word2idx:
                    continue
                trained_embeddings[token] = w2v_model[token].tolist()
                if self.word_embed_dim is None:
                    self.word_embed_dim = len(list(trained_embeddings[token]))
        elif kernel == 'kv':
            import pickle

            with open(embedding_path, 'rb') as fin:
                word_dict = pickle.load(fin)
                for token in word_dict:
                    if token not in self.word2idx:
                        continue
                    trained_embeddings[token] = word_dict[token]
                    if self.word_embed_dim is None:
                        self.word_embed_dim = len(list(trained_embeddings[token]))
        else:
            raise NotImplementedError("Not support embedding kernel {}.".format(kernel))

        filtered_tokens = trained_embeddings.keys()

        self.word2idx = OrderedDict()
        self.id2token = OrderedDict()
        for token in self.init_tokens:
            self.add_word(token, cnt=0)
        for token in filtered_tokens:
            self.add_word(token, cnt=0)

        # load embeddings
        self.word_embeddings = np.random.rand(self.word_size(), self.word_embed_dim)
        for token in self.word2idx.keys():
            if token in trained_embeddings:
                self.word_embeddings[self.get_word_idx(token)] = trained_embeddings[token] 

# 需要导入模块: from gensim.models.word2vec import Word2Vec [as 别名]
# 或者: from gensim.models.word2vec.Word2Vec import load [as 别名]
def load(self, filename):
		'''Load parameters values form a file
		'''
		self.w2v_model = Word2Vec.load(filename)