Python cPickle.load方法代码示例

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def test_read_backward_compatibility():
    """Test backwards compatibility with a pickled file that's created with Python 2.7.3,
    Numpy 1.7.1_ahl2 and Pandas 0.14.1
    """
    fname = path.join(path.dirname(__file__), "data", "test-data.pkl")

    # For newer versions; verify that unpickling fails when using cPickle
    if PANDAS_VERSION >= LooseVersion("0.16.1"):
        if sys.version_info[0] >= 3:
            with pytest.raises(UnicodeDecodeError), open(fname) as fh:
                cPickle.load(fh)
        else:
            with pytest.raises(TypeError), open(fname) as fh:
                cPickle.load(fh)

    # Verify that PickleStore() uses a backwards compatible unpickler.
    store = PickleStore()

    with open(fname) as fh:
        # PickleStore compresses data with lz4
        version = {'blob': compressHC(fh.read())}
    df = store.read(sentinel.arctic_lib, version, sentinel.symbol)

    expected = pd.DataFrame(range(4), pd.date_range(start="20150101", periods=4))
    assert (df == expected).all().all() 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def load_data(self, FLAGS):
        self.FLAGS = FLAGS
        with open(settings.CATES) as f:
            cates = json.load(f)
        text2cate = {c['text']: c['cate_id'] for c in cates}
        self.num_samples = 0
        self.labels = [[] for i in range(self.num_classes)]
        with open(FLAGS.dataset_dir, 'rb') as f:
            all = cPickle.load(f)
        for image, text in all:
            label = text2cate.get(text)
            assert label is not None
            if label is None or label >= settings.NUM_CHAR_CATES:
                self.labels[settings.NUM_CHAR_CATES].append(image)
            else:  # label < settings.NUM_CHAR_CATES:
                self.labels[label].append(image)
            self.num_samples += 1
        for label in self.labels:
            assert 0 < len(label)
        # self.preview() 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def test_function_dump():
    v = theano.tensor.vector()
    fct1 = theano.function([v], v + 1)

    try:
        tmpdir = tempfile.mkdtemp()
        fname = os.path.join(tmpdir, 'test_function_dump.pkl')
        theano.function_dump(fname, [v], v + 1)
        with open(fname, 'rb') as f:
            l = pickle.load(f)
    finally:
        if tmpdir is not None:
            shutil.rmtree(tmpdir)

    fct2 = theano.function(**l)
    x = [1, 2, 3]
    assert numpy.allclose(fct1(x), fct2(x)) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def sample(args):
    with open(os.path.join(args.save_dir, 'config.pkl'), 'rb') as f:
        saved_args = cPickle.load(f)
    with open(os.path.join(args.save_dir, 'chars_vocab.pkl'), 'rb') as f:
        chars, vocab = cPickle.load(f)
    #Use most frequent char if no prime is given
    if args.prime == '':
        args.prime = chars[0]
    model = Model(saved_args, training=False)
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(tf.global_variables())
        ckpt = tf.train.get_checkpoint_state(args.save_dir)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
            data = model.sample(sess, chars, vocab, args.n, args.prime,
                               args.sample).encode('utf-8')
            print(data.decode("utf-8")) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def read_dataset(data_dir):
    pickle_filename = "lamem.pickle"
    pickle_filepath = os.path.join(data_dir, pickle_filename)
    if not os.path.exists(pickle_filepath):
        utils.maybe_download_and_extract(data_dir, DATA_URL, is_tarfile=True)
        lamem_folder = (DATA_URL.split("/")[-1]).split(os.path.extsep)[0]
        result = {'images': create_image_lists(os.path.join(data_dir, lamem_folder))}
        print ("Pickling ...")
        with open(pickle_filepath, 'wb') as f:
            pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
    else:
        print ("Found pickle file!")

    with open(pickle_filepath, 'rb') as f:
        result = pickle.load(f)
        training_records = result['images']
        del result

    return training_records 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def _load_image_set_index(self, ext='json'):
        """
        Load the indexes listed in this dataset's image set file.
        """
        if cfg.LIMIT_RAM:
            if ext == 'json':
                path = pjoin(cfg.SPLIT_DIR, 'densecap_splits.json')
                with open(path, 'r') as f:
                    # NOTE: the return index has entries with INT type
                    image_index = json.load(f)[self._image_set]
                    print ("loading splits from {}".format(path))
            elif ext == 'txt':
                path = pjoin(cfg.SPLIT_DIR, '%s.txt' % self._image_set)
                with open(path, 'r') as f:
                    image_index = [line.strip() for line in f.readlines()]
                print ("loading splits from {}".format(path))
        else:
            image_index = [key for key in self._gt_regions]

        print("Number of examples: {}".format(len(image_index)))
        return image_index 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def from_snapshot(self, sess, sfile, nfile):
        print('Restoring model snapshots from {:s}'.format(sfile))
        self.saver.restore(sess, sfile)
        print('Restored.')
        # Needs to restore the other hyper-parameters/states for training, (TODO xinlei) I have
        # tried my best to find the random states so that it can be recovered exactly
        # However the Tensorflow state is currently not available
        with open(nfile, 'rb') as fid:
            st0 = pickle.load(fid)
            cur = pickle.load(fid)
            perm = pickle.load(fid)
            cur_val = pickle.load(fid)
            perm_val = pickle.load(fid)
            last_snapshot_iter = pickle.load(fid)

            np.random.set_state(st0)
            self.data_layer._cur = cur
            self.data_layer._perm = perm
            self.data_layer_val._cur = cur_val
            self.data_layer_val._perm = perm_val

        return last_snapshot_iter 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def read_data_files(self, subset='train'):
    """Reads from data file and returns images and labels in a numpy array."""
    assert self.data_dir, ('Cannot call `read_data_files` when using synthetic '
                           'data')
    if subset == 'train':
      filenames = [os.path.join(self.data_dir, 'data_batch_%d' % i)
                   for i in xrange(1, 6)]
    elif subset == 'validation':
      filenames = [os.path.join(self.data_dir, 'test_batch')]
    else:
      raise ValueError('Invalid data subset "%s"' % subset)

    inputs = []
    for filename in filenames:
      with gfile.Open(filename, 'r') as f:
        inputs.append(cPickle.load(f))
    # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
    # input format.
    all_images = np.concatenate(
        [each_input['data'] for each_input in inputs]).astype(np.float32)
    all_labels = np.concatenate(
        [each_input['labels'] for each_input in inputs])
    return all_images, all_labels 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def load_embeddings(lang="en", task="embeddings", type="cw", normalize=False):
  """Return a word embeddings object for `lang` and of type `type`

  Args:
    lang (string): language code.
    task (string): parameters that define task.
    type (string): skipgram, cw, cbow ...
    noramlized (boolean): returns noramlized word embeddings vectors.
  """
  src_dir = "_".join((type, task)) if type else task
  p = locate_resource(src_dir, lang)
  e = Embedding.load(p)
  if type == "cw":
    e.apply_expansion(CaseExpander)
    e.apply_expansion(DigitExpander)
  if type == "sgns":
    e.apply_expansion(CaseExpander)
  if type == "ue":
    e.apply_expansion(CaseExpander)
  if normalize:
    e.normalize_words(inplace=True)
  return e 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def _init_suffixes(self, filename):
        filename = path.join(self.data_dirname, filename)
        with open(filename, 'rb') as f:
            return pickle.load(f) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def load(self, filename):
        """Load counters to file"""
        try:
            with open(filename, 'rb') as fp:
                self.counters = cPickle.load(fp)
        except:
            logging.debug("can't load counter from file: %s", filename)
            return False
        return True 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def load_cifar_batch(fpath, label_key='labels'):
    """Internal utility for parsing CIFAR data.

    # Arguments
        fpath: path the file to parse.
        label_key: key for label data in the retrieve
            dictionary.

    # Returns
        A tuple `(data, labels)`.
    """
    f = utils.o_gfile(fpath, 'rb')
    if sys.version_info < (3,):
        d = cPickle.load(f)
    else:
        d = cPickle.load(f, encoding='bytes')
        # decode utf8
        d_decoded = {}
        for k, v in d.items():
            d_decoded[k.decode('utf8')] = v
        d = d_decoded
    f.close()
    data = d['data']
    labels = d[label_key]

    data = data.reshape(data.shape[0], 3, 32, 32)
    return data, labels 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def _load_dsprites(self, opts):
        """Load data from dsprites dataset

        """
        logging.debug('Loading dsprites')
        data_dir = _data_dir(opts)
        # pylint: disable=invalid-name
        # Let us use all the bad variable names!
        tr_X = None
        tr_Y = None
        te_X = None
        te_Y = None

        data_file = os.path.join(data_dir, 'dsprites.npz')
        X = np.load(data_file)['imgs']
        X = X[:, :, :, None]

        seed = 123
        np.random.seed(seed)
        np.random.shuffle(X)
        np.random.seed()

        self.data_shape = (64, 64, 1)
        test_size = 10000

        self.data = Data(opts, X[:-test_size])
        self.test_data = Data(opts, X[-test_size:])
        self.num_points = len(self.data)

        logging.debug('Loading Done.') 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def loadobject(path):
    with open(path, 'rb') as input:
        obj = pickle.load(input)
    return obj 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import load [as 别名]
def __init__(self, cache_filename):
        if os.path.exists(cache_filename):
            with open(cache_filename, 'rb') as f:
                self.old2new, self.new2old = cPickle.load(f)
        else:
            self.old2new, self.new2old = FilenameMapper.create_filename_map(cache_filename)