本文整理汇总了Python中six.moves.cPickle.HIGHEST_PROTOCOL属性的典型用法代码示例。如果您正苦于以下问题:Python cPickle.HIGHEST_PROTOCOL属性的具体用法?Python cPickle.HIGHEST_PROTOCOL怎么用?Python cPickle.HIGHEST_PROTOCOL使用的例子?那么, 这里精选的属性代码示例或许可以为您提供帮助。您也可以进一步了解该属性所在类six.moves.cPickle
的用法示例。
在下文中一共展示了cPickle.HIGHEST_PROTOCOL属性的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_pickle_chunk_V1_read
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def test_pickle_chunk_V1_read():
data = {'foo': b'abcdefghijklmnopqrstuvwxyz'}
version = {'_id': sentinel._id,
'blob': '__chunked__'}
coll = Mock()
arctic_lib = Mock()
datap = compressHC(cPickle.dumps(data, protocol=cPickle.HIGHEST_PROTOCOL))
data_1 = datap[0:5]
data_2 = datap[5:]
coll.find.return_value = [{'data': Binary(data_1),
'symbol': 'sentinel.symbol',
'segment': 0},
{'data': Binary(data_2),
'symbol': 'sentinel.symbol',
'segment': 1},
]
arctic_lib.get_top_level_collection.return_value = coll
ps = PickleStore()
assert(data == ps.read(arctic_lib, version, sentinel.symbol))
示例2: test_pickle_store_future_version
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def test_pickle_store_future_version():
data = {'foo': b'abcdefghijklmnopqrstuvwxyz'}
version = {'_id': sentinel._id,
'blob': '__chunked__VERSION_ONE_MILLION'}
coll = Mock()
arctic_lib = Mock()
datap = compressHC(cPickle.dumps(data, protocol=cPickle.HIGHEST_PROTOCOL))
data_1 = datap[0:5]
data_2 = datap[5:]
coll.find.return_value = [{'data': Binary(data_1),
'symbol': 'sentinel.symbol',
'segment': 0},
{'data': Binary(data_2),
'symbol': 'sentinel.symbol',
'segment': 1},
]
arctic_lib.get_top_level_collection.return_value = coll
ps = PickleStore()
with pytest.raises(UnsupportedPickleStoreVersion) as e:
ps.read(arctic_lib, version, sentinel.symbol)
assert('unsupported version of pickle store' in str(e.value))
示例3: save_pkl
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def save_pkl(self):
"""
Dump this object into its `key_pkl` file.
May raise a cPickle.PicklingError if such an exception is raised at
pickle time (in which case a warning is also displayed).
"""
# Note that writing in binary mode is important under Windows.
try:
with open(self.key_pkl, 'wb') as f:
pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)
except pickle.PicklingError:
_logger.warning("Cache leak due to unpickle-able key data %s",
self.keys)
os.remove(self.key_pkl)
raise
示例4: read_dataset
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def read_dataset(data_dir):
pickle_filename = "lamem.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_tarfile=True)
lamem_folder = (DATA_URL.split("/")[-1]).split(os.path.extsep)[0]
result = {'images': create_image_lists(os.path.join(data_dir, lamem_folder))}
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['images']
del result
return training_records
示例5: maybe_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + '.pickle'
dataset_names.append(set_filename)
if os.path.exists(set_filename) and not force:
# You may override by setting force=True.
print('%s already present - Skipping pickling.' % set_filename)
else:
print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
开发者ID:PacktPublishing,项目名称:Neural-Network-Programming-with-TensorFlow,代码行数:20,代码来源:1_prepare_pickle_200.py
示例6: maybe_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + '.pickle'
dataset_names.append(set_filename)
if os.path.exists(set_filename) and not force:
print('%s already present - Skipping pickling.' % set_filename)
else:
print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
#pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
print(pickle.HIGHEST_PROTOCOL)
pickle.dump(dataset, f, 2)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
开发者ID:PacktPublishing,项目名称:Neural-Network-Programming-with-TensorFlow,代码行数:21,代码来源:prepare_notmnist.py
示例7: main
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def main(params):
info = json.load(open(params['dict_json'], 'r'))
imgs = json.load(open(params['input_json'], 'r'))
itow = info['ix_to_word']
wtoi = {w:i for i,w in itow.items()}
wtod = {w:i+1 for w,i in info['wtod'].items()} # word to detection
# dtoi = {w:i+1 for i,w in enumerate(wtod.keys())} # detection to index
dtoi = wtod
wtol = info['wtol']
itod = {i:w for w,i in dtoi.items()}
# imgs = imgs['images']
ngram_idxs, ref_len = build_dict(imgs, info, wtoi, wtod, dtoi, wtol, itod, params)
# cPickle.dump({'document_frequency': ngram_words, 'ref_len': ref_len}, open(params['output_pkl']+'-words.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)
cPickle.dump({'document_frequency': ngram_idxs, 'ref_len': ref_len}, open(params['output_pkl']+'-idxs.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)
示例8: main
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def main(params):
det_train_path = 'data/coco/annotations/instances_train2014.json'
det_val_path = 'data/coco/annotations/instances_val2014.json'
coco_det_train = COCO(det_train_path)
coco_det_val = COCO(det_val_path)
info = json.load(open(params['dict_json'], 'r'))
imgs = json.load(open(params['input_json'], 'r'))
itow = info['ix_to_word']
wtoi = {w:i for i,w in itow.items()}
wtod = {w:i+1 for w,i in info['wtod'].items()} # word to detection
dtoi = {w:i+1 for i,w in enumerate(wtod.keys())} # detection to index
wtol = info['wtol']
ctol = {c:i+1 for i, c in enumerate(coco_det_train.cats.keys())}
# imgs = imgs['images']
ngram_idxs, ref_len = build_dict(imgs, info, wtoi, wtod, dtoi, wtol, ctol, coco_det_train, coco_det_val, params)
# cPickle.dump({'document_frequency': ngram_words, 'ref_len': ref_len}, open(params['output_pkl']+'-words.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)
cPickle.dump({'document_frequency': ngram_idxs, 'ref_len': ref_len}, open(params['output_pkl']+'-idxs.p','w'), protocol=cPickle.HIGHEST_PROTOCOL)
示例9: recursive_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def recursive_pickle(top_obj):
"""
Recursively pickle all of the given objects subordinates, starting with
the deepest first. **Very** handy for debugging pickling issues, but
also very slow (as it literally pickles each object in turn).
Handles circular object references gracefully.
"""
objs = depth_getter(top_obj)
# sort by depth then by nest_info
objs = sorted(six.itervalues(objs), key=lambda val: (-val[0], val[2]))
for _, obj, location in objs:
# print('trying %s' % location)
try:
pickle.dump(obj, BytesIO(), pickle.HIGHEST_PROTOCOL)
except Exception as err:
print(obj)
print('Failed to pickle %s. \n Type: %s. Traceback '
'follows:' % (location, type(obj)))
raise
示例10: maybe_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + '.pickle'
dataset_names.append(set_filename)
if os.path.exists(set_filename) and not force:
# You may override by setting force=True.
print('%s already present - Skipping pickling.' % set_filename)
else:
print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
示例11: maybe_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + pickle_extension
dataset_names.append(folder)
if os.path.exists(set_filename) and not force:
# You may override by setting force=True.
print('%s already present - Skipping pickling.' % set_filename)
else:
# print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
Pickle.dump(dataset, f, Pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
示例12: save_snapshot
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def save_snapshot(self, filename=None):
"""
Save a snapshot of current process to file
Warning: this is not thread safe, do not use with multithread program
Args:
- filename: target file to save snapshot
Returns:
- Bool
"""
if not filename:
filename = self.get_config_filename("snapshot")
snapshot = self.take_snapshot()
if not snapshot:
return False
# dump to file
fd = open(filename, "wb")
pickle.dump(snapshot, fd, pickle.HIGHEST_PROTOCOL)
fd.close()
return True
示例13: maybe_pickle
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + '.pickle'
dataset_names.append(set_filename)
if os.path.exists(set_filename) and not force:
# You may override by setting force=True.
print('%s already present - Skipping pickling.' % set_filename)
else:
print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
示例14: read_dataset
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def read_dataset(data_dir):
pickle_filename = "MITSceneParsing.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
SceneParsing_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
result = create_image_lists(os.path.join(data_dir, SceneParsing_folder))
print ("> [SPD] Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("> [SPD] Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['validation']
del result
return training_records, validation_records
示例15: read_dataset
# 需要导入模块: from six.moves import cPickle [as 别名]
# 或者: from six.moves.cPickle import HIGHEST_PROTOCOL [as 别名]
def read_dataset(data_dir):
pickle_filename = "MITSceneParsing.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
SceneParsing_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
result = create_image_lists(os.path.join(data_dir, SceneParsing_folder))
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['validation']
del result
return training_records, validation_records