Python cPickle.loads方法代码示例

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def predict(self, data_root, model_root, test_root, test_div, out_path, readable=False):
        meta_path = os.path.join(data_root, 'meta')
        meta = cPickle.loads(open(meta_path, 'rb').read())

        model_fname = os.path.join(model_root, 'model.h5')
        self.logger.info('# of classes(train): %s' % len(meta['y_vocab']))
        model = load_model(model_fname,
                           custom_objects={'top1_acc': top1_acc})

        test_path = os.path.join(test_root, 'data.h5py')
        test_data = h5py.File(test_path, 'r')

        test = test_data[test_div]
        batch_size = opt.batch_size
        pred_y = []
        test_gen = ThreadsafeIter(self.get_sample_generator(test, batch_size, raise_stop_event=True))
        total_test_samples = test['uni'].shape[0]
        with tqdm.tqdm(total=total_test_samples) as pbar:
            for chunk in test_gen:
                total_test_samples = test['uni'].shape[0]
                X, _ = chunk
                _pred_y = model.predict(X)
                pred_y.extend([np.argmax(y) for y in _pred_y])
                pbar.update(X[0].shape[0])
        self.write_prediction_result(test, pred_y, meta, out_path, readable=readable) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def load(fname):
    """Load an embedding dump generated by `save`"""

    content = _open(fname).read()
    if PY2:
      state = pickle.loads(content)
    else:
      state = pickle.loads(content, encoding='latin1')
    voc, vec = state
    if len(voc) == 2:
      words, counts = voc
      word_count = dict(zip(words, counts))
      vocab = CountedVocabulary(word_count=word_count)
    else:
      vocab = OrderedVocabulary(voc)
    return Embedding(vocabulary=vocab, vectors=vec) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def verify_tee(n, original, seed):
    try:
        state = random.getstate()
        iterators = list(tee(original, n=n))
        results = [[] for _ in range(n)]
        exhausted = [False] * n
        while not all(exhausted):
            # Upper argument of random.randint is inclusive. Argh.
            i = random.randint(0, n - 1)
            if not exhausted[i]:
                if len(results[i]) == len(original):
                    assert_raises(StopIteration, next, iterators[i])
                    assert results[i] == original
                    exhausted[i] = True
                else:
                    if random.randint(0, 1):
                        iterators[i] = cPickle.loads(
                            cPickle.dumps(iterators[i]))
                    elem = next(iterators[i])
                    results[i].append(elem)
    finally:
        random.setstate(state) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def test_xrange():
    yield assert_equal, list(xrange(10)), list(_xrange(10))
    yield assert_equal, list(xrange(10, 15)), list(_xrange(10, 15))
    yield assert_equal, list(xrange(10, 20, 2)), list(_xrange(10, 20, 2))
    yield assert_equal, list(xrange(5, 1, -1)), list(_xrange(5, 1, -1))
    yield (assert_equal, list(xrange(5, 55, 3)),
           list(cPickle.loads(cPickle.dumps(_xrange(5, 55, 3)))))
    yield assert_equal, _xrange(5).index(4), 4
    yield assert_equal, _xrange(5, 9).index(6), 1
    yield assert_equal, _xrange(8, 24, 3).index(11), 1
    yield assert_equal, _xrange(25, 4, -5).index(25), 0
    yield assert_equal, _xrange(28, 7, -7).index(14), 2
    yield assert_raises, ValueError, _xrange(2, 9, 2).index, 3
    yield assert_raises, ValueError, _xrange(2, 20, 2).index, 9
    yield assert_equal, _xrange(5).count(5), 0
    yield assert_equal, _xrange(5).count(4), 1
    yield assert_equal, _xrange(4, 9).count(4), 1
    yield assert_equal, _xrange(3, 9, 2).count(4), 0
    yield assert_equal, _xrange(3, 9, 2).count(5), 1
    yield assert_equal, _xrange(3, 9, 2).count(20), 0
    yield assert_equal, _xrange(9, 3).count(5), 0
    yield assert_equal, _xrange(3, 10, -1).count(5), 0
    yield assert_equal, _xrange(10, 3, -1).count(5), 1
    yield assert_equal, _xrange(10, 0, -2).count(6), 1
    yield assert_equal, _xrange(10, -1, -3).count(7), 1 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def minimize(self, model, data, max_evals, notebook_name=None):
        global best_model_yaml, best_model_weights

        trials_list = self.compute_trials(
            model, data, max_evals, notebook_name)

        best_val = 1e7
        for trials in trials_list:
            for trial in trials:
                val = trial.get('result').get('loss')
                if val < best_val:
                    best_val = val
                    best_model_yaml = trial.get('result').get('model')
                    best_model_weights = trial.get('result').get('weights')

        best_model = model_from_yaml(best_model_yaml)
        best_model.set_weights(pickle.loads(best_model_weights))

        return best_model 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def best_models(self, nb_models, model, data, max_evals):
        trials_list = self.compute_trials(model, data, max_evals)
        num_trials = sum(len(trials) for trials in trials_list)
        if num_trials < nb_models:
            nb_models = len(trials_list)
        scores = []
        for trials in trials_list:
            scores = scores + [trial.get('result').get('loss')
                               for trial in trials]
        cut_off = sorted(scores, reverse=True)[nb_models - 1]
        model_list = []
        for trials in trials_list:
            for trial in trials:
                if trial.get('result').get('loss') >= cut_off:
                    model = model_from_yaml(trial.get('result').get('model'))
                    model.set_weights(pickle.loads(
                        trial.get('result').get('weights')))
                    model_list.append(model)
        return model_list 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def test_pickling(self, backend_config):
        x_data, = self.generate_inputs()

        link = self.create_link(self.generate_params())
        link.to_device(backend_config.device)

        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)

        y = link(x)
        y_data1 = y.data
        del x, y
        pickled = pickle.dumps(link, -1)
        del link
        link = pickle.loads(pickled)
        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)
        y = link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def check_pickling(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data1 = y.data

        del x, y

        pickled = pickle.dumps(self.link, -1)
        del self.link
        self.link = pickle.loads(pickled)

        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def test_pickling(self):
        try:
            features = numpy.arange(360, dtype='uint16').reshape((10, 36))
            h5file = h5py.File('file.hdf5', mode='w')
            h5file['features'] = features
            split_dict = {'train': {'features': (0, 10, None, '.')}}
            h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
            dataset = cPickle.loads(
                cPickle.dumps(H5PYDataset(h5file, which_sets=('train',))))
            # Make sure _out_of_memory_{open,close} accesses
            # external_file_handle rather than _external_file_handle
            dataset._out_of_memory_open()
            dataset._out_of_memory_close()
            assert dataset.data_sources is None
        finally:
            os.remove('file.hdf5') 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def get_project_settings():
    if ENVVAR not in os.environ:
        project = os.environ.get('SCRAPY_PROJECT', 'default')
        init_env(project)

    settings = Settings()
    settings_module_path = os.environ.get(ENVVAR)
    if settings_module_path:
        settings.setmodule(settings_module_path, priority='project')

    # XXX: remove this hack
    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
    if pickled_settings:
        settings.setdict(pickle.loads(pickled_settings), priority='project')

    # XXX: deprecate and remove this functionality
    env_overrides = {k[7:]: v for k, v in os.environ.items() if
                     k.startswith('SCRAPY_')}
    if env_overrides:
        settings.setdict(env_overrides, priority='project')

    return settings 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def _read_data(self, spider, request):
        key = self._request_key(request)
        try:
            ts = self.db.Get(key + b'_time')
        except KeyError:
            return  # not found or invalid entry

        if 0 < self.expiration_secs < time() - float(ts):
            return  # expired

        try:
            data = self.db.Get(key + b'_data')
        except KeyError:
            return  # invalid entry
        else:
            return pickle.loads(data) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def execute_request(self, request):
        # noinspection PyBroadException
        try:
            # Use pickle to read the binary data.
            data_object = pickle.loads(request)
        except Exception:  # pickle.loads is document as raising any type of exception, so have to catch them all.
            self.__logger.warn(
                "Could not parse incoming metric line from graphite pickle server, ignoring",
                error_code="graphite_monitor/badUnpickle",
            )
            return

        try:
            # The format should be [[ metric [ timestamp, value]] ... ]
            for (metric, datapoint) in data_object:
                value = float(datapoint[1])
                orig_timestamp = float(datapoint[0])
                self.__logger.emit_value(
                    metric, value, extra_fields={"orig_time": orig_timestamp}
                )
        except ValueError:
            self.__logger.warn(
                "Could not parse incoming metric line from graphite pickle server, ignoring",
                error_code="graphite_monitor/badPickleLine",
            ) 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def test_pickle_invoke():
    data = pwny.pickle_invoke(func_to_invoke, 8)
    assert cPickle.loads(data) == 8 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def test_pickle_func():
    def func_to_invoke_2(a):
        return a

    data = pwny.pickle_func(func_to_invoke_2, 8)

    del func_to_invoke_2

    assert cPickle.loads(data) == 8 

# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import loads [as 别名]
def query(self, *args, **kwargs):
        """
        Generic query method.

        In reality, your storage class would have its own query methods,

        Performs a Mongo find on the Marketdata index metadata collection.
        See:
        http://api.mongodb.org/python/current/api/pymongo/collection.html
        """
        for x in self._collection.find(*args, **kwargs):
            x['stuff'] = cPickle.loads(x['stuff'])
            del x['_id'] # Remove default unique '_id' field from doc 
            yield Stuff(**x)