Python LSHForest.partial_fit方法代码示例

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]
def test_partial_fit():
    """Checks whether inserting array is consitent with fitted data.

    `partial_fit` method should set all attribute values correctly.
    """
    n_samples = 12
    n_samples_partial_fit = 3
    n_features = 2
    rng = np.random.RandomState(42)
    X = rng.rand(n_samples, n_features)
    X_partial_fit = rng.rand(n_samples_partial_fit, n_features)

    lshf = LSHForest()

    # Test unfitted estimator
    lshf.partial_fit(X)
    assert_array_equal(X, lshf._fit_X)

    lshf.fit(X)

    # Insert wrong dimension
    assert_raises(ValueError, lshf.partial_fit,
                  np.random.randn(n_samples_partial_fit, n_features - 1))

    lshf.partial_fit(X_partial_fit)

    # size of _input_array = samples + 1 after insertion
    assert_equal(lshf._fit_X.shape[0],
                 n_samples + n_samples_partial_fit)
    # size of original_indices_[1] = samples + 1
    assert_equal(len(lshf.original_indices_[0]),
                 n_samples + n_samples_partial_fit)
    # size of trees_[1] = samples + 1
    assert_equal(len(lshf.trees_[1]),
                 n_samples + n_samples_partial_fit)

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]
def cal_acc(pack_file, stat_file, feature_dim):
    f = open(stat_file, 'w')
    f.write('train_pic_num'+'\t'+'person_name'+'\t'+'acc'+'\n')
    pic_num = range(1, max_person_num)
    for num in pic_num:
        all_train_data, all_train_label, all_valid_data, all_valid_label = split_train_valid(pack_file, train_pic_num=num, feature_dim=feature_dim)
        lshf = LSHForest(n_estimators=20, n_candidates=200, n_neighbors=5)

        for index in range(len(all_train_data)):
            try:
                if all_train_data[index] == None:
                    continue
                lshf.partial_fit(all_train_data[index], all_train_label[index])
            except:
                traceback.print_exc()
                continue
        # 对于每个人,分别统计准确率
        person_acc_dic = {}     # 准确的个数
        person_all_dic = {}     # 总的个数
        filter_num = 0
        all_num = 0
        for index in range(len(all_valid_data)):
            try:
                if all_valid_data[index] == None:
                    continue
                all_find_distance, all_find_index = lshf.kneighbors(all_valid_data[index], n_neighbors=5, return_distance=True)
                cos_sim = cosine_similarity(all_valid_data[index], all_train_data[all_find_index[0, 0]])
                label = all_train_label[all_find_index[0, 0]]
                # if cos_sim > sim_threshold:
                if True:
                    if label == all_valid_label[index]:
                        person_acc_dic[label] = person_acc_dic.get(label, 0) + 1
                        person_all_dic[label] = person_all_dic.get(label, 0) + 1
                    else:
                        person_all_dic[label] = person_all_dic.get(label, 0) + 1
                else:
                    filter_num += 1
                all_num += 1
            except:
                print all_valid_label[index]
                continue
        print 'train_num :', num, 'filter_rate: ', (filter_num * 1.0 / all_num)
        for person in person_all_dic:
            all_num = person_all_dic[person]
            right_num = person_acc_dic.get(person, 0)
            f.write('\t'.join(map(str, [num, person, (right_num * 1.0 /  all_num)]))+'\n')

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]
def cal_recall(pack_file, stat_file, feature_dim):
    # f_model = open('verf.txt', 'w')
    f = open(stat_file, 'w')
    f.write('train_pic_num'+'\t'+'person_name'+'\t'+'recall'+'\n')
    pic_num = range(1, max_person_num)
    for num in pic_num:
        all_train_data, all_train_label, all_valid_data, all_valid_label = split_train_valid(pack_file, train_pic_num=num, feature_dim=feature_dim)
        lshf = LSHForest(n_estimators=20, n_candidates=200, n_neighbors=5)
        for index in range(len(all_train_data)):
            try:
                if all_train_data[index] == None:
                    continue
                lshf.partial_fit(all_train_data[index], all_train_label[index])
            except:
                continue
        # 对于每个人,分别统计准确率
        person_find_dic = {}     # 准确的个数
        person_all_dic = {}     # 总的个数
        for index in range(len(all_valid_data)):
            try:
                if all_valid_data[index] == None:
                    continue
                all_find_distance, all_find_index = lshf.kneighbors(all_valid_data[index], n_neighbors=5, return_distance=True)
                cos_sim = cosine_similarity(all_valid_data[index], all_train_data[all_find_index[0, 0]])
                label = all_train_label[all_find_index[0, 0]]
                real_label = all_valid_label[index]
                # if cos_sim > sim_threshold:
                if True:
                    if label == real_label:
                        # f_model.write('0'+'\t'+str(cos_sim)+'\n')
                        person_find_dic[real_label] = person_find_dic.get(real_label, 0) + 1
                        person_all_dic[real_label] = person_all_dic.get(real_label, 0) + 1
                    else:
                        # f_model.write('1' + '\t' + str(cos_sim) + '\n')
                        person_all_dic[real_label] = person_all_dic.get(real_label, 0) + 1
            except:
                print all_valid_label[index]
                continue
        print 'train_num :', num
        for person in person_all_dic:
            all_num = person_all_dic[person]
            right_num = person_find_dic.get(person, 0)
            f.write('\t'.join(map(str, [num, person, (right_num * 1.0 /  all_num)]))+'\n')

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]
class Search():
    def __init__(self, model_type, n_estimators=20, n_candidates=200, n_neighbors=10):
        self.lshf = LSHForest(n_estimators=n_estimators, n_candidates=n_candidates, n_neighbors=n_neighbors)

        if model_type == 'rgb_small':
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.small.rgb.deepid.model'
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.small.rgb.deepid.weight'
            self.part_func = None
            self.pic_shape = (50, 50, 3)
            self.feature_dim = 1024
        elif model_type == 'rgb_big':
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.big.rgb.deepid.weight'
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.big.rgb.deepid.model'
            self.part_func = None
            self.pic_shape = (128, 128, 3)
        elif model_type == 'rgb_small_right':
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.small.rgb.right_eye.deepid.weight'
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.small.rgb.right_eye.deepid.model'
            self.part_func = get_right_eye
            self.pic_shape = (50, 50, 3)
        elif model_type == 'rgb_small_left':
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.small.rgb.left_eye.deepid.weight'
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.small.rgb.left_eye.deepid.model'
            self.part_func = get_left_eye
            self.pic_shape = (50, 50, 3)
        elif model_type == 'rgb_small_nose':
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.all.rgb.nose.deepid.weight'
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.all_data.all.rgb.nose.deepid.model'
            self.part_func = get_nose
            self.pic_shape = (50, 50, 3)
        elif model_type == 'new_shape':
            self.deepid_model_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.new_shape.rgb.deepid.model'
            self.deepid_weight_file = '/data/liubo/face/vgg_face_dataset/model/vgg_face.small_data.new_shape.rgb.deepid.weight'
            self.pic_shape = (156, 124, 3)
            self.feature_dim = 256
            self.part_func = None
        self.model, self.get_Conv_FeatureMap = load_deepid_model(self.deepid_model_file, self.deepid_weight_file)
        self.all_label = None
        self.all_feature_data = None


    def extract_pic_feature(self, pic_data, batch_size=128, feature_dim=1024):
        pic_feature = np.zeros(shape=(pic_data.shape[0], feature_dim))
        batch_num = pic_data.shape[0] / batch_size
        for index in range(batch_num):
            # pic_feature[index*batch_size:(index+1)*batch_size, :] = \
            #     self.get_Conv_FeatureMap([pic_data[index*batch_size:(index+1)*batch_size], 0])[0]
            pic_feature[index*batch_size:(index+1)*batch_size, :] = \
                self.get_Conv_FeatureMap([np.transpose(pic_data[index*batch_size:(index+1)*batch_size], (0, 3, 1, 2)), 0])[0]

        if batch_num*batch_size < pic_data.shape[0]:
            # pic_feature[batch_num*batch_size:, :] = \
            #     self.get_Conv_FeatureMap([pic_data[batch_num*batch_size:], 0])[0]
            pic_feature[batch_num*batch_size:, :] = \
                self.get_Conv_FeatureMap([np.transpose(pic_data[batch_num*batch_size:], (0, 3, 1, 2)), 0])[0]
        return pic_feature


    def train_all_data(self, vgg_folder, person_num=100, batch_person_num=20, pic_num=10):
        # 取前pic_num张图片加入到LSH Forest,其余图片用于判断准确率
        for index in range(0+train_person_start_index, person_num+train_person_start_index, batch_person_num):
            if index == 0+train_person_start_index:
                pic_data, all_label = load_batch_train_data(vgg_folder, shape=self.pic_shape, start_person_index=index,
                                 pic_num=pic_num, batch_num=batch_person_num, is_train=True, part_func=self.part_func)
                all_data_feature = self.extract_pic_feature(pic_data, feature_dim=self.feature_dim)
                self.lshf.fit(all_data_feature, all_label)

            else:
                pic_data, this_label = load_batch_train_data(vgg_folder, start_person_index=index, pic_num=pic_num,
                                shape=self.pic_shape, batch_num=batch_person_num,is_train=True, part_func=self.part_func)
                all_label = np.row_stack((np.reshape(all_label, (all_label.shape[0], 1)),
                                          np.reshape(this_label, (this_label.shape[0],1))))
                pic_data_feature = self.extract_pic_feature(pic_data, feature_dim=self.feature_dim)
                all_data_feature = np.row_stack((pic_data_feature, all_data_feature))
                self.lshf.partial_fit(pic_data_feature, this_label)
        self.all_label = all_label
        self.all_feature_data = all_data_feature
        logging.info(' '.join(map(str, ['self.all_label.shape :', self.all_label.shape])))


    def partical_fit(self, pic_data, this_label):
        '''
            增量训练, 样本比较小, 直接
        :param data:
        :param label:
        :return:
        '''
        pic_data_feature = self.extract_pic_feature(pic_data, feature_dim=self.feature_dim)
        self.lshf.partial_fit(pic_data_feature, this_label)
        self.all_label = np.row_stack((np.reshape(self.all_label, (self.all_label.shape[0], 1)),
                                          np.reshape(this_label, (this_label.shape[0],1))))


    def find_k_neighbors(self, pic_data):
        pic_data_feature = self.extract_pic_feature(pic_data, feature_dim=self.feature_dim)
        distances, indices = self.lshf.kneighbors(pic_data_feature, n_neighbors=1)
        predict_label = self.all_label[indices][:, 0, 0]
        return predict_label


#.........这里部分代码省略.........

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]

#.........这里部分代码省略.........
                        log_file.write('\t'.join(map(str, ['no add file :', pic_path]))+'\n')
                        traceback.print_exc()
                        continue
                    add_num += 1
        end = time.time()
        if add_num > 0:
            self.load_time = end
            log_file.write('\t'.join(map(str, ['self.load_time', self.load_time]))+'\n')
            log_file.write('\t'.join(map(str, ['add pic num :', add_num,
                                               'Dynamic increase time :', (end - start)]))+'\n')
            log_file.close()
        else:
            log_file.close()


    def add_one_new_pic(self, pic_path, label):
        try:
            # 读入数据时已经转换成需要的尺寸
            im_feature = extract_feature_from_file(pic_path)
            self.add_one_pic(im_feature, label)
            return True
        except:
            traceback.print_exc()
            return False


    def add_one_pic(self, one_pic_feature, pic_label):
        '''
            将一个图像的特征加入到LSH Forest,同时将对应的标签加入到self.all_labels
            :param pic_feature: array shape :(1,1024)
            :param pic_label: (1,)
            :return:
        '''
        self.lshf.partial_fit(one_pic_feature.reshape(1, FEATURE_DIM), pic_label)
        self.all_labels.append(pic_label)
        self.all_pic_feature.append(np.reshape(one_pic_feature, newshape=(1, one_pic_feature.size)))


    def find_k_neighbors_with_lsh(self, one_pic_feature):
        '''
            :param one_pic_feature: 图像特征
            :return: 需要返回neighbors的特征, 用于计算pariwise
        '''
        try:
            tmp = self.lshf.kneighbors(one_pic_feature.reshape(1, FEATURE_DIM), n_neighbors=self.n_neighbors, return_distance=True)
            neighbors_label = np.asarray(self.all_labels)[tmp[1][0]]
            neighbors_feature = np.asarray(self.all_pic_feature)[tmp[1][0]]
            pair_score_list = []
            cos_sim_list = []
            for index in range(len(neighbors_feature)):
                pair_score = pw.cosine_similarity(neighbors_feature[index].reshape(1, FEATURE_DIM),
                                     one_pic_feature.reshape(1, FEATURE_DIM))[0][0]
                cos_sim_list.append(pair_score)
                pair_score_list.append(verification_model.predict(pair_score))
            result = zip(cos_sim_list, pair_score_list, neighbors_label)
            # result = self.filter_result(result)
            # result.sort(key=lambda x:x[0], reverse=True)
            return result
        except:
            return None


    def filter_result(self, result):
        '''
            :param result: [(cos_sim, same_person_result, label),
                            (cos_sim, same_person_result, label),

# 需要导入模块: from sklearn.neighbors import LSHForest [as 别名]
# 或者: from sklearn.neighbors.LSHForest import partial_fit [as 别名]

#.........这里部分代码省略.........
            # 读入数据时已经转换成需要的尺寸
            result = self.extract_pic_feature(pic_path)
            if result == None:
                return False
            face_pic, pic_feature = result
            self.add_one_pic(pic_feature, label)
            pic_name = os.path.split(pic_path)[1]
            this_person_pic_folder = os.path.join(self.all_pic_data_folder, label)
            this_person_feature_folder = os.path.join(self.all_pic_feature_data_folder, label)
            if not os.path.exists(this_person_pic_folder):
                os.makedirs(this_person_pic_folder)
            if not os.path.exists(this_person_feature_folder):
                os.makedirs(this_person_feature_folder)
            # 直接存储图片对应的特征, 同时保存图片文件
            this_pic_feature_name = os.path.join(this_person_feature_folder, pic_name + '.p')
            msgpack_numpy.dump(pic_feature, open(this_pic_feature_name, 'wb'))
            this_pic_face_name = os.path.join(this_person_pic_folder, pic_name + '.jpg')
            cv2.imwrite(this_pic_face_name, face_pic)
            log_file.write('\t'.join(map(str, [pic_path, this_pic_face_name]))+'\n')
            return True
        except:
            traceback.print_exc()
            return False


    def add_one_pic(self, one_pic_feature, pic_label):
        '''
            将一个图像的特征加入到LSH Forest,同时将对应的标签加入到self.all_labels
            :param pic_feature: array shape :(1,1024)
            :param pic_label: (1,)
            :return:
        '''
        one_pic_feature = np.asarray(one_pic_feature)
        self.lshf.partial_fit(one_pic_feature.reshape(1, FEATURE_DIM), pic_label)
        self.all_labels.append(pic_label)
        self.all_pic_feature.append(np.reshape(one_pic_feature, newshape=(1, one_pic_feature.size)))


    def find_k_neighbors_with_lsh(self, one_pic_feature):
        '''
            :param one_pic_feature: 图像特征
            :return: 需要返回neighbors的特征,用于计算pariwise
        '''
        try:
            one_pic_feature = np.asarray(one_pic_feature)
            tmp = self.lshf.kneighbors(one_pic_feature.reshape(1, FEATURE_DIM), n_neighbors=self.n_neighbors, return_distance=True)
            neighbors_label = np.asarray(self.all_labels)[tmp[1][0]]
            neighbors_feature = np.asarray(self.all_pic_feature)[tmp[1][0]]
            pair_score_list = []
            cos_sim_list = []
            for index in range(len(neighbors_feature)):
                pair_score = pw.cosine_similarity(neighbors_feature[index].reshape(1, FEATURE_DIM),
                                     one_pic_feature.reshape(1, FEATURE_DIM))[0][0]
                cos_sim_list.append(pair_score)
                pair_score_list.append(verification_model.predict(pair_score))
            result = zip(cos_sim_list, pair_score_list, neighbors_label)
            # result = self.filter_result(result)
            # result.sort(key=lambda x:x[0], reverse=True)
            return result
        except:
            return None


    def filter_result(self, result):
        '''
            :param result: [(cos_sim, same_person_result, label),