Python utils.load_pkl函数代码示例

def test_luna3d():
    # path = '/mnt/sda3/data/kaggle-lung/lunapred/luna_scan_v3_dice-20170131-173443/'
    path = '/mnt/sda3/data/kaggle-lung/lunapred_el/luna_scan_v3_dice-20170201-231707/'
    files = os.listdir(path)
    print files
    x, y, p = [], [], []
    for f in files:
        if 'in' in f:
            x.append(f)
        elif 'tgt' in f:
            y.append(f)
        else:
            p.append(f)
    x = sorted(x)
    y = sorted(y)
    p = sorted(p)
    for xf, yf, pf in zip(x, y, p):
        x_batch = utils.load_pkl(path + xf)
        pred_batch = utils.load_pkl(path + pf)
        y_batch = utils.load_pkl(path + yf)
        print xf
        print yf
        print pf
        # plot_2d_animation(x_batch[0], y_batch[0], pred_batch[0])
        plot_slice_3d_3(x_batch[0,0],y_batch[0,0],pred_batch[0,0],0,'aa')

def load_pretrained_model(l_in):


    l = conv3d(l_in, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)
    l = feat_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)
    l = feat_red(l)
    l = inrn_v2(l)

    l = feat_red(l)

    l = dense(l, 128, name='dense_fpr')

    l_out = nn.layers.DenseLayer(l, num_units=2,
                                 W=nn.init.Constant(0.),
                                 nonlinearity=nn.nonlinearities.softmax)


    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/ikorshun/","luna_c3-20170226-174919.pkl"))
    nn.layers.set_all_param_values(l_out, metadata['param_values'])

    return nn.layers.get_all_layers(l_out)[-3]

    def __init__(self, data_path, batch_size, transform_params, patient_ids=None, labels_path=None,
                 slice2roi_path=None, full_batch=False, random=True, infinite=False, view='sax',
                 data_prep_fun=data.transform_norm_rescale, **kwargs):

        if patient_ids:
            self.patient_paths = []
            for pid in patient_ids:
                self.patient_paths.append(data_path + '/%s/study/' % pid)
        else:
            self.patient_paths = glob.glob(data_path + '/*/study/')

        self.slice_paths = [sorted(glob.glob(p + '/%s_*.pkl' % view)) for p in self.patient_paths]
        self.slice_paths = list(itertools.chain(*self.slice_paths))
        self.slicepath2pid = {}
        for s in self.slice_paths:
            self.slicepath2pid[s] = int(utils.get_patient_id(s))

        self.nsamples = len(self.slice_paths)
        self.batch_size = batch_size
        self.rng = np.random.RandomState(42)
        self.full_batch = full_batch
        self.random = random
        self.infinite = infinite
        self.id2labels = data.read_labels(labels_path) if labels_path else None
        self.transformation_params = transform_params
        self.data_prep_fun = data_prep_fun
        self.slice2roi = utils.load_pkl(slice2roi_path) if slice2roi_path else None

def build_model():
    l_in = nn.layers.InputLayer((None, ) + p_transform['patch_size'])
    l_dim = nn.layers.DimshuffleLayer(l_in, pattern=[0,'x',1,2,3])
    l_target = nn.layers.InputLayer((None, 1))

    l = conv3d(l_dim, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)
    l = feat_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)
    l = feat_red(l)
    l = inrn_v2(l)

    l = feat_red(l) 

    l_out = dense(l, 128)

    # l_out = nn.layers.DenseLayer(l, num_units=2,
    #                              W=nn.init.Constant(0.),
    #                              nonlinearity=nn.nonlinearities.softmax)

    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/ikorshun/","luna_c3-20170226-174919.pkl"))

    for i in range(-20,0):
        print metadata['param_values'][i].shape

    nn.layers.set_all_param_values(l_out, metadata['param_values'][:-2])

    return namedtuple('Model', ['l_in', 'l_out', 'l_target'])(l_in, l_out, l_target)

def load_pretrained_model(l_in):


    l = conv3d(l_in, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2_red(l)

    l = dense(drop(l), 128)

    l_out = nn.layers.DenseLayer(l, num_units=10,
                                 W=nn.init.Orthogonal(),
                                 b=nn.init.Constant(0.1),
                                 nonlinearity=nn.nonlinearities.softmax)



    metadata = utils.load_pkl(os.path.join("/mnt/storage/metadata/dsb3/models/eavsteen/","t_el_0-20170321-013339.pkl"))
    nn.layers.set_all_param_values(l_out, metadata['param_values'])

    return nn.layers.get_all_layers(l_out)[-3]

def build_segmentation_model(l_in):
    metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
    metadata_path = utils.find_model_metadata(metadata_dir, patch_segmentation_config.__name__.split('.')[-1])
    metadata = utils.load_pkl(metadata_path)

    model = patch_segmentation_config.build_model(l_in=l_in, patch_size=p_transform['patch_size'])
    nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
    return model

def test3():
    image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
    id2mm_shape = utils.load_pkl(image_dir + '/pid2mm.pkl')
    s = [(key, value) for (key, value) in sorted(id2mm_shape.items(), key=lambda x: x[1][0])]
    for i in xrange(5):
        print s[i]
    print '--------------------------'
    for i in xrange(1,6):
        print s[-i]

def load_pretrained_model(l_in):

    l = conv3d(l_in, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2_red(l)

    l = dense(drop(l), 512)

    d_final_layers = {}
    final_layers = []
    unit_ptr = 0

    for obj_idx, obj_name in enumerate(cfg_prop.order_objectives):
        ptype = cfg_prop.property_type[obj_name]

        if ptype == 'classification':
            num_units = len(cfg_prop.property_bin_borders[obj_name])
            l_fin = nn.layers.DenseLayer(l, num_units=num_units,
                     W=nn.init.Orthogonal(),
                     b=nn.init.Constant(cfg_prop.init_values_final_units[obj_name]),
                     nonlinearity=nn.nonlinearities.softmax, name='dense_softmax_'+ptype+'_'+obj_name)

        elif ptype == 'continuous':
            l_fin = nn.layers.DenseLayer(l, num_units=1,
                    W=nn.init.Orthogonal(),
                    b=nn.init.Constant(cfg_prop.init_values_final_units[obj_name]),
                    nonlinearity=nn.nonlinearities.softplus, name='dense_softplus_'+ptype+'_'+obj_name)

        elif ptype == 'bounded_continuous':
            l_fin = nn.layers.DenseLayer(l, num_units=1,
                    W=nn.init.Orthogonal(),
                    b=nn.init.Constant(cfg_prop.init_values_final_units[obj_name]),
                    nonlinearity=nn.nonlinearities.sigmoid, name='dense_sigmoid_'+ptype+'_'+obj_name)
        else:
            raise

        d_final_layers[obj_name] = l_fin
        final_layers.append(l_fin)

    l_out = nn.layers.ConcatLayer(final_layers, name = 'final_concat_layer')


    metadata = utils.load_pkl(os.path.join('/home/frederic/kaggle-dsb3/dsb/storage/metadata/dsb3/models/eavsteen/',"r_elias_28-20170331-230303.pkl"))
    nn.layers.set_all_param_values(l_out, metadata['param_values'])

    features = d_final_layers['malignancy']
    print 'features layer', features.name

    return features

  def load_weight_from_pkl(self, cpu_mode=False):
    with tf.variable_scope('load_pred_from_pkl'):
      self.w_input = {}
      self.w_assign_op = {}

      for name in self.w.keys():
        self.w_input[name] = tf.placeholder('float32', self.w[name].get_shape().as_list(), name=name)
        self.w_assign_op[name] = self.w[name].assign(self.w_input[name])

    for name in self.w.keys():
      self.w_assign_op[name].eval({self.w_input[name]: load_pkl(os.path.join(self.weight_dir, "%s.pkl" % name))})

    self.update_target_q_network()

def load_pretrained_model(l_in):

    l = conv3d(l_in, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2_red(l)

    l = drop(l, name='can_dropout')
    l = dense(l, 512, name='can_dense')

    final_layers = []
    for obj_idx, obj_name in enumerate(cfg_prop.order_objectives):
        ptype = cfg_prop.property_type[obj_name]
        if ptype == 'classification':
            num_units = len(cfg_prop.property_bin_borders[obj_name])
            l_fin = nn.layers.DenseLayer(l, num_units=num_units,
                     W=nn.init.Orthogonal(),
                     b=nn.init.Constant(cfg_prop.init_values_final_units[obj_name]),
                     nonlinearity=nn.nonlinearities.softmax, name='dense_'+ptype+'_'+obj_name)

        elif ptype == 'continuous':
            l_fin = nn.layers.DenseLayer(l, num_units=1,
                    W=nn.init.Orthogonal(),
                    b=nn.init.Constant(cfg_prop.init_values_final_units[obj_name]),
                    nonlinearity=nn.nonlinearities.softplus, name='dense_'+ptype+'_'+obj_name)

        else:
          raise

        final_layers.append(l_fin)

    l_out = nn.layers.ConcatLayer(final_layers, name = 'final_concat_layer')


    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/eavsteen/","r_elias_10-20170328-003348.pkl"))
    nn.layers.set_all_param_values(l_out, metadata['param_values'])

    features = nn.layers.get_all_layers(l_out)[(-2-len(final_layers))]
    print 'features layer', features.name

    return features

def test_luna3d():
    image_dir = utils.get_dir_path('analysis', pathfinder.METADATA_PATH)
    image_dir = image_dir + '/test_luna/'
    utils.auto_make_dir(image_dir)

    id2zyxd = utils_lung.read_luna_annotations(pathfinder.LUNA_LABELS_PATH)

    luna_data_paths = [
        '/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.mhd']

    candidates = utils.load_pkl(
        '/mnt/sda3/data/kaggle-lung/luna_test_patient/1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474.pkl')

    candidates = candidates[:4]
    print candidates
    print '--------------'
    print id2zyxd['1.3.6.1.4.1.14519.5.2.1.6279.6001.877026508860018521147620598474']

    for k, p in enumerate(luna_data_paths):
        id = os.path.basename(p).replace('.mhd', '')
        print id
        img, origin, pixel_spacing = utils_lung.read_mhd(p)
        lung_mask = lung_segmentation.segment_HU_scan_ira(img)
        print np.min(lung_mask), np.max(lung_mask)
        x, annotations_tf, tf_matrix, lung_mask_out = data_transforms.transform_scan3d(data=img,
                                                                                       pixel_spacing=pixel_spacing,
                                                                                       p_transform=p_transform,
                                                                                       luna_annotations=candidates,
                                                                                       p_transform_augment=None,
                                                                                       luna_origin=origin,
                                                                                       lung_mask=lung_mask,
                                                                                       world_coord_system=False)

        print np.min(lung_mask_out), np.max(lung_mask_out)

        plot_slice_3d_2(x, lung_mask_out, 0, id)
        plot_slice_3d_2(x, lung_mask_out, 1, id)
        plot_slice_3d_2(x, lung_mask_out, 2, id)

        # for zyxd in annotations_tf:
        #     plot_slice_3d_2(x, lung_mask_out, 0, id, idx=zyxd)
        #     plot_slice_3d_2(x, lung_mask_out, 1, id, idx=zyxd)
        #     plot_slice_3d_2(x, lung_mask_out, 2, id, idx=zyxd)

        for i in xrange(136, x.shape[1]):
            plot_slice_3d_2(x, lung_mask_out, 1, str(id) + str(i), idx=np.array([200, i, 200]))

def build_model():
    metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
    metadata_path = utils.find_model_metadata(metadata_dir, patch_class_config.__name__.split('.')[-1])
    metadata = utils.load_pkl(metadata_path)

    print 'Build model'
    model = patch_class_config.build_model()
    all_layers = nn.layers.get_all_layers(model.l_out)
    num_params = nn.layers.count_params(model.l_out)
    print '  number of parameters: %d' % num_params
    print string.ljust('  layer output shapes:', 36),
    print string.ljust('#params:', 10),
    print 'output shape:'
    for layer in all_layers:
        name = string.ljust(layer.__class__.__name__, 32)
        num_param = sum([np.prod(p.get_value().shape) for p in layer.get_params()])
        num_param = string.ljust(num_param.__str__(), 10)
        print '    %s %s %s' % (name, num_param, layer.output_shape)

    nn.layers.set_all_param_values(model.l_out, metadata['param_values'])
    return model

def load_pretrained_model(l_in):

    l = conv3d(l_in, 64)
    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2(l)

    l = inrn_v2_red(l)
    l = inrn_v2_red(l)

    l = dense(drop(l), 512)

    l = nn.layers.DenseLayer(l,1,nonlinearity=nn.nonlinearities.sigmoid, W=nn.init.Orthogonal(),
                b=nn.init.Constant(0))


    metadata = utils.load_pkl(os.path.join("/home/eavsteen/dsb3/storage/metadata/dsb3/models/eavsteen/","r_fred_malignancy_2-20170328-230443.pkl"))
    nn.layers.set_all_param_values(l, metadata['param_values'])

    return l

def evaluate_trained(config, state, channel):
    config_path = config.load_trained.from_path + 'model_config.pkl'
    epoch = config.load_trained.epoch
    params_path = config.load_trained.from_path + 'model_params_e%d.pkl'%(epoch) 
    assert config_path is not None
    assert params_path is not None
    assert os.path.isfile(params_path)
    assert os.path.isfile(config_path)
    print 'load the config options from the best trained model'
    used_config = utils.load_pkl(config_path)
    action = config.load_trained.action
    assert action == 1
    from_path = config.load_trained.from_path
    epoch = config.load_trained.epoch
    save_model_path = config.load_trained.from_path
    set_config(config, used_config)
    config.load_trained.action = action
    config.load_trained.from_path = from_path
    config.load_trained.epoch = epoch
    config.save_model_path = save_model_path
    
    model_type = config.model
    # set up automatically some fields in config
    if config.dataset.signature == 'MNIST_binary_russ':
        config[model_type].n_in = 784
        config[model_type].n_out = 784
    # Also copy back from config into state.
    for key in config:
        setattr(state, key, config[key])

    print 'Model Type: %s'%model_type
    print 'Host:    %s' % socket.gethostname()
    print 'Command: %s' % ' '.join(sys.argv)
    
    print 'initializing data engine'
    input_dtype = 'float32'
    target_dtype = 'int32'
    data_engine = None
    deep_orderless_bernoulli_nade.evaluate_trained(state, data_engine, params_path, channel)

def build_model():
    l_in = nn.layers.InputLayer((None, n_candidates_per_patient, 1,) + p_transform['patch_size'])
    l_in_rshp = nn.layers.ReshapeLayer(l_in, (-1, 1,) + p_transform['patch_size'])
    l_target = nn.layers.InputLayer((batch_size,))

    base_n_filters = 128
    l = conv_prelu_layer(l_in_rshp, n_filters=base_n_filters)
    l = conv_prelu_layer(l, n_filters=base_n_filters)
    l = conv_prelu_layer(l, n_filters=base_n_filters)

    l = max_pool3d(l)

    l = conv_prelu_layer(l, n_filters=base_n_filters)
    l = conv_prelu_layer(l, n_filters=base_n_filters)
    l = conv_prelu_layer(l, n_filters=base_n_filters)
    l_enc = conv_prelu_layer(l, n_filters=base_n_filters)

    num_units_dense = 512
    l_d01 = dense_prelu_layer(l, num_units=512)
    l_d01 = nn.layers.ReshapeLayer(l_d01, (-1, n_candidates_per_patient, num_units_dense))
    l_d02 = dense_prelu_layer(l_d01, num_units=512)
    l_out = nn.layers.DenseLayer(l_d02, num_units=2,
                                 W=nn.init.Constant(0.),
                                 b=np.array([np.log((1397. - 362) / 1398), np.log(362. / 1397)], dtype='float32'),
                                 nonlinearity=nn.nonlinearities.softmax)

    metadata_dir = utils.get_dir_path('models', pathfinder.METADATA_PATH)
    metadata_path = utils.find_model_metadata(metadata_dir, 'luna_p8a1')
    metadata = utils.load_pkl(metadata_path)
    for p, pv in zip(nn.layers.get_all_params(l_enc), metadata['param_values']):
        if p.get_value().shape != pv.shape:
            raise ValueError("mismatch: parameter has shape %r but value to "
                             "set has shape %r" %
                             (p.get_value().shape, pv.shape))
        p.set_value(pv)

    return namedtuple('Model', ['l_in', 'l_out', 'l_target'])(l_in, l_out, l_target)