Python cPickle.load方法代码示例

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def _augment_images_worker(self, augseq, queue_source, queue_result):
        """Worker function that endlessly queries the source queue (input
        batches), augments batches in it and sends the result to the output
        queue."""
        while True:
            # wait for a new batch in the source queue and load it
            batch_str = queue_source.get()
            batch = pickle.loads(batch_str)

            # augment the batch
            if batch.images is not None and batch.keypoints is not None:
                augseq_det = augseq.to_deterministic()
                batch.images_aug = augseq_det.augment_images(batch.images)
                batch.keypoints_aug = augseq_det.augment_keypoints(batch.keypoints)
            elif batch.images is not None:
                batch.images_aug = augseq.augment_images(batch.images)
            elif batch.keypoints is not None:
                batch.keypoints_aug = augseq.augment_keypoints(batch.keypoints)

            # send augmented batch to output queue
            queue_result.put(pickle.dumps(batch, protocol=-1)) 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def register(self, name, serializer):
        """Register ``serializer`` object under ``name``.

        Raises :class:`AttributeError` if ``serializer`` in invalid.

        .. note::

            ``name`` will be used as the file extension of the saved files.

        :param name: Name to register ``serializer`` under
        :type name: ``unicode`` or ``str``
        :param serializer: object with ``load()`` and ``dump()``
            methods

        """
        # Basic validation
        getattr(serializer, 'load')
        getattr(serializer, 'dump')

        self._serializers[name] = serializer 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def from_snapshot(self, sfile, nfile):
    print('Restoring model snapshots from {:s}'.format(sfile))
    self.net.load_state_dict(torch.load(str(sfile)))
    print('Restored.')
    # Needs to restore the other hyper-parameters/states for training, (TODO xinlei) I have
    # tried my best to find the random states so that it can be recovered exactly
    # However the Tensorflow state is currently not available
    with open(nfile, 'rb') as fid:
      st0 = pickle.load(fid)
      cur = pickle.load(fid)
      perm = pickle.load(fid)
      cur_val = pickle.load(fid)
      perm_val = pickle.load(fid)
      last_snapshot_iter = pickle.load(fid)

      np.random.set_state(st0)
      self.data_layer._cur = cur
      self.data_layer._perm = perm
      self.data_layer_val._cur = cur_val
      self.data_layer_val._perm = perm_val

    return last_snapshot_iter 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def preprocess_omniglot():
  """Download and prepare raw Omniglot data.

  Downloads the data from GitHub if it does not exist.
  Then load the images, augment with rotations if desired.
  Resize the images and write them to a pickle file.
  """

  maybe_download_data()

  directory = TRAIN_DIR
  write_file = DATA_FILE_FORMAT % 'train'
  num_labels = write_datafiles(
      directory, write_file, resize=True, rotate=TRAIN_ROTATIONS,
      new_width=IMAGE_NEW_SIZE, new_height=IMAGE_NEW_SIZE)

  directory = TEST_DIR
  write_file = DATA_FILE_FORMAT % 'test'
  write_datafiles(directory, write_file, resize=True, rotate=TEST_ROTATIONS,
                  new_width=IMAGE_NEW_SIZE, new_height=IMAGE_NEW_SIZE,
                  first_label=num_labels) 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def extract_mnist_data(filename, num_images, image_size, pixel_depth):
  """
  Extract the images into a 4D tensor [image index, y, x, channels].

  Values are rescaled from [0, 255] down to [-0.5, 0.5].
  """
  # if not os.path.exists(file):
  if not tf.gfile.Exists(filename+".npy"):
    with gzip.open(filename) as bytestream:
      bytestream.read(16)
      buf = bytestream.read(image_size * image_size * num_images)
      data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
      data = (data - (pixel_depth / 2.0)) / pixel_depth
      data = data.reshape(num_images, image_size, image_size, 1)
      np.save(filename, data)
      return data
  else:
    with tf.gfile.Open(filename+".npy", mode='r') as file_obj:
      return np.load(file_obj) 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def _load_data(self):
        """
        Load data only if the present data is not checkpointed, else, just load the checkpointed data

        :return: None
        """
        self.mapper = Mapper()
        self.mapper.generate_vocabulary(self.review_summary_file)
        self.X_fwd, self.X_bwd, self.Y = self.mapper.get_tensor(reverseflag=True)
        # Store all the mapper values in a dict for later recovery
        self.mapper_dict = dict()
        self.mapper_dict['seq_length'] = self.mapper.get_seq_length()
        self.mapper_dict['vocab_size'] = self.mapper.get_vocabulary_size()
        self.mapper_dict['rev_map'] = self.mapper.get_reverse_map()
        # Split into test and train data
        self._split_train_tst() 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def _load_data(self):
        """
        Load data only if the present data is not checkpointed, else, just load the checkpointed data

        :return: None
        """
        self.mapper = Mapper()
        self.mapper.generate_vocabulary(self.review_summary_file)
        self.X, self.Y = self.mapper.get_tensor()
        # Store all the mapper values in a dict for later recovery
        self.mapper_dict = dict()
        self.mapper_dict['seq_length'] = self.mapper.get_seq_length()
        self.mapper_dict['vocab_size'] = self.mapper.get_vocabulary_size()
        self.mapper_dict['rev_map'] = self.mapper.get_reverse_map()
        # Split into test and train data
        self._split_train_tst() 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def from_saved(cls, file_name, change_params={}):
        """
        Initializes a new network from saved data.
        file_name (str): model is loaded from tf_save/file_name.ckpt
        """
        with open(io.tf_save_path + file_name + '.pkl', 'rb') as f:
            params = pickle.load(f)
        params['load_file'] = file_name
        for p in change_params:
            params[p] = change_params[p]
        print params
        return cls(**params)

    #########################################
    #        Private helper functions       #
    ######################################### 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def load(validation_size_p, file_name):
    """
    Params:
    validation_size_p: percentage of data to be used for validation
    file_name (str): File containing the data
    """
    f = gzip.open(io.data_path + file_name + ".plk.gz", 'rb')
    data, states, projectors = cPickle.load(f)
    data = np.array(data)
    states = np.array(states)
    train_val_separation = int(len(data) * (1 - validation_size_p / 100.))
    training_data = data[:train_val_separation]
    training_states = states[:train_val_separation]
    validation_data = data[train_val_separation:]
    validation_states = states[train_val_separation:]
    f.close()
    return (training_data, validation_data, training_states, validation_states, projectors) 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def read_pickle(self,filename):
        try:
            import cPickle as pickle
        except ImportError:
            import pickle

        in_f = open(filename,"rb")

        tabversion = pickle.load(in_f)
        if tabversion != __tabversion__:
            raise VersionError("yacc table file version is out of date")
        self.lr_method = pickle.load(in_f)
        signature      = pickle.load(in_f)
        self.lr_action = pickle.load(in_f)
        self.lr_goto   = pickle.load(in_f)
        productions    = pickle.load(in_f)

        self.lr_productions = []
        for p in productions:
            self.lr_productions.append(MiniProduction(*p))

        in_f.close()
        return signature

    # Bind all production function names to callable objects in pdict 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def _prune(self):
        entries = self._list_dir()
        if len(entries) > self._threshold:
            now = time()
            for idx, fname in enumerate(entries):
                remove = False
                f = None
                try:
                    try:
                        f = open(fname, 'rb')
                        expires = pickle.load(f)
                        remove = expires <= now or idx % 3 == 0
                    finally:
                        if f is not None:
                            f.close()
                except Exception:
                    pass
                if remove:
                    try:
                        os.remove(fname)
                    except (IOError, OSError):
                        pass 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def loadFiles(self,pklfile,outspecfile):
        """ loads pkl and outspec files
        Args:
            pklfile (string) - full filepath to pkl file to load
            outspecfile (string) - fille filepath to outspec.json file
        Return:
            DRM (dict) - a dict containing seed, DRM, system
            outspec (dict) - a dict containing input instructions
        """
        try:
            with open(pklfile, 'rb') as f:#load from cache
                DRM = pickle.load(f)
        except:
            print('Failed to open pklfile %s'%pklfile)
            pass
        try:
            with open(outspecfile, 'rb') as g:
                outspec = json.load(g)
        except:
            print('Failed to open outspecfile %s'%outspecfile)
            pass
        return DRM, outspec 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def read_all(run_dir):
    """
    Helper function that reads in all pkl files from an nsemble directory
    generated by run_ipcluster_ensemble
    
    Args:
        run_dir (string):
            Absolute path to run directory 
    
    Returns:
        allres (list):
            List of all pkl file contents in run_dir
    """
    
    pklfiles = glob.glob(os.path.join(run_dir,'*.pkl'))

    allres = []

    for counter,f in enumerate(pklfiles):
        print("%d/%d"%(counter,len(pklfiles)))
        with open(f, 'rb') as g:
            res = pickle.load(g, encoding='latin1')
        allres.append(res)
        del res # this avoids memory leaks when loading many pickle files
    return allres 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def read_json_lines(file_path):
    print("reading data...")
    with open(file_path, "r") as f:
        lines = []
        value_err_cnt = 0
        for l in tqdm(f.readlines()):
            try:
                loaded_l = json.loads(l.strip("\n"))
                lines.append(loaded_l)
            except ValueError as e:
                value_err_cnt += 1
                continue
    return lines


# def load_pickle(file_path):
#     with open(file_path, "r") as f:
#         return pickle.load(f) 

# 需要导入模块: import cPickle [as 别名]
# 或者: from cPickle import load [as 别名]
def load_dynamic_contour(template_flame_path='None', contour_embeddings_path='None', static_embedding_path='None', angle=0):
    template_mesh = Mesh(filename=template_flame_path)
    contour_embeddings_path = contour_embeddings_path
    dynamic_lmks_embeddings = np.load(contour_embeddings_path, allow_pickle=True).item()
    lmk_face_idx_static, lmk_b_coords_static = load_static_embedding(static_embedding_path)
    lmk_face_idx_dynamic = dynamic_lmks_embeddings['lmk_face_idx'][angle]
    lmk_b_coords_dynamic = dynamic_lmks_embeddings['lmk_b_coords'][angle]
    dynamic_lmks = mesh_points_by_barycentric_coordinates(template_mesh.v, template_mesh.f, lmk_face_idx_dynamic, lmk_b_coords_dynamic)
    static_lmks = mesh_points_by_barycentric_coordinates(template_mesh.v, template_mesh.f, lmk_face_idx_static, lmk_b_coords_static)
    total_lmks = np.vstack([dynamic_lmks, static_lmks])

    # Visualization of the pose dependent contour on the template mesh
    vertex_colors = np.ones([template_mesh.v.shape[0], 4]) * [0.3, 0.3, 0.3, 0.8]
    tri_mesh = trimesh.Trimesh(template_mesh.v, template_mesh.f,
                               vertex_colors=vertex_colors)
    mesh = pyrender.Mesh.from_trimesh(tri_mesh)
    scene = pyrender.Scene()
    scene.add(mesh)
    sm = trimesh.creation.uv_sphere(radius=0.005)
    sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0]
    tfs = np.tile(np.eye(4), (len(total_lmks), 1, 1))
    tfs[:, :3, 3] = total_lmks
    joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
    scene.add(joints_pcl)
    pyrender.Viewer(scene, use_raymond_lighting=True)