本文整理汇总了Python中oio.blob.client.BlobClient.chunk_delete方法的典型用法代码示例。如果您正苦于以下问题：Python BlobClient.chunk_delete方法的具体用法？Python BlobClient.chunk_delete怎么用？Python BlobClient.chunk_delete使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类oio.blob.client.BlobClient的用法示例。
在下文中一共展示了BlobClient.chunk_delete方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestDupContent
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
#.........这里部分代码省略.........
content = self.content_factory.new(self.container_id, "titi",
len(data), "TWOCOPIES")
self.assertEqual(type(content), DupContent)
# set bad url for position 1
for chunk in content.chunks.filter(pos=1):
chunk.url = "http://127.0.0.1:9/DEADBEEF"
self.assertRaises(Exception, content.upload, StringIO.StringIO(data))
for chunk in content.chunks.exclude(pos=1):
self.assertRaises(NotFound,
self.blob_client.chunk_head, chunk.url)
def _new_content(self, stgpol, data, broken_pos_list=[]):
old_content = self.content_factory.new(self.container_id, "titi",
len(data), stgpol)
self.assertEqual(type(old_content), DupContent)
old_content.upload(StringIO.StringIO(data))
broken_chunks_info = {}
for pos, idx in broken_pos_list:
c = old_content.chunks.filter(pos=pos)[idx]
meta, stream = self.blob_client.chunk_get(c.url)
if pos not in broken_chunks_info:
broken_chunks_info[pos] = {}
broken_chunks_info[pos][idx] = {
"url": c.url,
"id": c.id,
"hash": c.hash,
"dl_meta": meta,
"dl_hash": md5_stream(stream)
}
self.blob_client.chunk_delete(c.url)
# get the new structure of the uploaded content
return (self.content_factory.get(
self.container_id, old_content.content_id), broken_chunks_info)
def _test_rebuild(self, stgpol, data_size, broken_pos_list,
full_rebuild_pos):
data = random_data(data_size)
content, broken_chunks_info = self._new_content(stgpol,
data, broken_pos_list)
rebuild_pos, rebuild_idx = full_rebuild_pos
rebuild_chunk_info = broken_chunks_info[rebuild_pos][rebuild_idx]
content.rebuild_chunk(rebuild_chunk_info["id"])
# get the new structure of the content
rebuilt_content = self.content_factory.get(self.container_id,
content.content_id)
self.assertEqual(type(rebuilt_content), DupContent)
# find the rebuilt chunk
for c in rebuilt_content.chunks.filter(pos=rebuild_pos):
if len(content.chunks.filter(id=c.id)) > 0:
# not the rebuilt chunk
# if this chunk is broken, it must not have been rebuilt
for b_c_i in broken_chunks_info[rebuild_pos].values():
if c.id == b_c_i["id"]:
with ExpectedException(NotFound):
_, _ = self.blob_client.chunk_get(c.url)
continue
meta, stream = self.blob_client.chunk_get(c.url)
self.assertEqual(meta["chunk_id"], c.id)示例2: BlobMoverWorker
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
#.........这里部分代码省略.........
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(mover_time).2f'
'%(mover_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'mover_time': mover_time,
'mover_rate': mover_time / (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
mover_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(mover_time).2f '
'%(mover_rate).2f' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'mover_time': mover_time,
'mover_rate': mover_time / elapsed
}
)
def safe_chunk_move(self, path):
try:
self.chunk_move(path)
except Exception as e:
self.errors += 1
self.logger.error('ERROR while moving chunk %s: %s', path, e)
self.passes += 1
def load_chunk_metadata(self, path):
with open(path) as f:
return read_chunk_metadata(f)
def chunk_move(self, path):
meta = self.load_chunk_metadata(path)
content_cid = meta['content_cid']
content_path = meta['content_path']
chunk_url = 'http://%s/%s' % \
(self.address, meta['chunk_id'])
try:
data = self.container_client.content_show(
cid=content_cid, path=content_path)
except exc.NotFound:
raise exc.OrphanChunk('Content not found')
current_chunk = None
notin = []
for c in data:
if c['pos'] == meta['chunk_pos']:
notin.append(c)
for c in notin:
if c['url'] == chunk_url:
current_chunk = c
notin.remove(c)
if not current_chunk:
raise exc.OrphanChunk('Chunk not found in content')
spare_data = {'notin': notin, 'broken': [current_chunk], 'size': 0}
spare_resp = self.container_client.content_spare(
cid=content_cid, path=content_path, data=spare_data)
new_chunk = spare_resp['chunks'][0]
self.blob_client.chunk_copy(
current_chunk['url'], new_chunk['id'])
old = [{'type': 'chunk',
'id': current_chunk['url'],
'hash': meta['chunk_hash'],
'size': int(meta['chunk_size'])}]
new = [{'type': 'chunk',
'id': new_chunk['id'],
'hash': meta['chunk_hash'],
'size': int(meta['chunk_size'])}]
update_data = {'old': old, 'new': new}
self.container_client.container_raw_update(
cid=content_cid, data=update_data)
self.blob_client.chunk_delete(current_chunk['url'])
self.logger.info(
'moved chunk %s to %s', current_chunk['url'], new_chunk['id'])示例3: Content
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
class Content(object):
def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.stgpol_args = stgpol_args
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient()
self.session = requests.Session()
self.content_id = self.metadata["id"]
self.stgpol_name = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.hash = self.metadata["hash"]
self.mime_type = self.metadata["mime-type"]
self.chunk_method = self.metadata["chunk-method"]
def _meta2_get_spare_chunk(self, chunks_notin, chunks_broken):
spare_data = {
"notin": ChunksHelper(chunks_notin, False).raw(),
"broken": ChunksHelper(chunks_broken, False).raw()
}
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id, content=self.content_id,
data=spare_data, stgpol=self.stgpol_name)
except ClientException as e:
raise exc.SpareChunkException("No spare chunk (%s)" % e.message)
url_list = []
for c in spare_resp["chunks"]:
url_list.append(c["id"])
return url_list
def _meta2_update_spare_chunk(self, current_chunk, new_url):
old = [{'type': 'chunk',
'id': current_chunk.url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
new = [{'type': 'chunk',
'id': new_url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
update_data = {'old': old, 'new': new}
self.container_client.container_raw_update(
cid=self.container_id, data=update_data)
def _meta2_create_object(self):
self.container_client.content_create(cid=self.container_id,
path=self.path,
content_id=self.content_id,
stgpol=self.stgpol_name,
size=self.length,
checksum=self.hash,
version=self.version,
chunk_method=self.chunk_method,
mime_type=self.mime_type,
data=self.chunks.raw())
def rebuild_chunk(self, chunk_id):
raise NotImplementedError()
def upload(self, stream):
try:
self._upload(stream)
except Exception as e:
for chunk in self.chunks:
try:
self.blob_client.chunk_delete(chunk.url)
except:
pass
raise e
def _upload(self, stream):
raise NotImplementedError()
def download(self):
raise NotImplementedError()示例4: Content
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
class Content(object):
def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
self.conf = conf
self.container_id = container_id
self.metadata = metadata
self.chunks = ChunksHelper(chunks)
self.stgpol_args = stgpol_args
self.logger = get_logger(self.conf)
self.cs_client = ConscienceClient(conf)
self.container_client = ContainerClient(self.conf)
self.blob_client = BlobClient()
self.session = requests.Session()
self.content_id = self.metadata["id"]
self.stgpol_name = self.metadata["policy"]
self.path = self.metadata["name"]
self.length = int(self.metadata["length"])
self.version = self.metadata["version"]
self.hash = self.metadata["hash"]
self.mime_type = self.metadata["mime-type"]
self.chunk_method = self.metadata["chunk-method"]
def _meta2_get_spare_chunk(self, chunks_notin, chunks_broken):
spare_data = {
"notin": ChunksHelper(chunks_notin, False).raw(),
"broken": ChunksHelper(chunks_broken, False).raw()
}
try:
spare_resp = self.container_client.content_spare(
cid=self.container_id, content=self.content_id,
data=spare_data, stgpol=self.stgpol_name)
except ClientException as e:
raise exc.SpareChunkException("No spare chunk (%s)" % e.message)
url_list = []
for c in spare_resp["chunks"]:
url_list.append(c["id"])
return url_list
def _meta2_update_spare_chunk(self, current_chunk, new_url):
old = [{'type': 'chunk',
'id': current_chunk.url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
new = [{'type': 'chunk',
'id': new_url,
'hash': current_chunk.hash,
'size': current_chunk.size,
'pos': current_chunk.pos,
'content': self.content_id}]
update_data = {'old': old, 'new': new}
self.container_client.container_raw_update(
cid=self.container_id, data=update_data)
def _meta2_create_object(self):
self.container_client.content_create(cid=self.container_id,
path=self.path,
content_id=self.content_id,
stgpol=self.stgpol_name,
size=self.length,
checksum=self.hash,
version=self.version,
chunk_method=self.chunk_method,
mime_type=self.mime_type,
data=self.chunks.raw())
def rebuild_chunk(self, chunk_id):
raise NotImplementedError()
def upload(self, stream):
try:
self._upload(stream)
except:
# Keep the stack trace
exc_info = sys.exc_info()
for chunk in self.chunks:
try:
self.blob_client.chunk_delete(chunk.url)
except:
self.logger.warn("Failed to delete %s", chunk.url)
# Raise with the original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
def _upload(self, stream):
raise NotImplementedError()
def download(self):
raise NotImplementedError()
def delete(self):
self.container_client.content_delete(cid=self.container_id,
path=self.path)
def move_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
#.........这里部分代码省略.........
示例5: TestRainContent
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
#.........这里部分代码省略.........
# set bad url for position 1
for chunk in content.chunks.filter(pos="1.p0"):
chunk.url = "http://127.0.0.1:9/DEADBEEF"
self.assertRaises(Exception, content.upload, StringIO.StringIO(data))
for chunk in content.chunks.exclude(pos="1.p0"):
self.assertRaises(NotFound,
self.blob_client.chunk_head, chunk.url)
def _test_rebuild(self, data_size, broken_pos_list):
data = os.urandom(data_size)
old_content = self.content_factory.new(self.container_id, "titi",
len(data), "RAIN")
self.assertEqual(type(old_content), RainContent)
old_content.upload(StringIO.StringIO(data))
# get the new structure of the uploaded content
uploaded_content = self.content_factory.get(self.container_id,
old_content.content_id)
old_info = {}
for pos in broken_pos_list:
old_info[pos] = {}
c = uploaded_content.chunks.filter(pos=pos)[0]
old_info[pos]["url"] = c.url
old_info[pos]["id"] = c.id
old_info[pos]["hash"] = c.hash
chunk_id_to_rebuild = c.id
meta, stream = self.blob_client.chunk_get(c.url)
old_info[pos]["dl_meta"] = meta
old_info[pos]["dl_hash"] = md5_stream(stream)
# delete the chunk
self.blob_client.chunk_delete(c.url)
# rebuild the broken chunks
uploaded_content.rebuild_chunk(chunk_id_to_rebuild)
# get the new structure of the content
rebuilt_content = self.content_factory.get(self.container_id,
uploaded_content.content_id)
self.assertEqual(type(rebuilt_content), RainContent)
for pos in broken_pos_list:
c = rebuilt_content.chunks.filter(pos=pos)[0]
rebuilt_meta, rebuilt_stream = self.blob_client.chunk_get(c.url)
self.assertEqual(rebuilt_meta["chunk_id"], c.id)
self.assertEqual(md5_stream(rebuilt_stream),
old_info[pos]["dl_hash"])
self.assertEqual(c.hash, old_info[pos]["hash"])
self.assertThat(c.url, NotEquals(old_info[pos]["url"]))
del old_info[pos]["dl_meta"]["chunk_id"]
del rebuilt_meta["chunk_id"]
self.assertEqual(rebuilt_meta, old_info[pos]["dl_meta"])
def test_content_0_byte_rebuild_pos_0_0(self):
self._test_rebuild(0, ["0.0"])
def test_content_0_byte_rebuild_pos_0_0_and_0_p0(self):
self._test_rebuild(0, ["0.0", "0.p0"])
def test_content_1_byte_rebuild_pos_0_0(self):
self._test_rebuild(1, ["0.0"])
def test_content_1_byte_rebuild_pos_0_p0(self):
self._test_rebuild(1, ["0.p0"])示例6: TestECContent
# 需要导入模块: from oio.blob.client import BlobClient [as 别名]
# 或者: from oio.blob.client.BlobClient import chunk_delete [as 别名]
#.........这里部分代码省略.........
self._test_create(1)
def test_create(self):
self._test_create(DAT_LEGIT_SIZE)
def _test_rebuild(self, data_size, broken_pos_list):
# generate test data
data = os.urandom(data_size)
# create initial content
old_content = self.content_factory.new(
self.container_id, self.content, len(data), self.stgpol)
# verify factory work as intended
self.assertEqual(type(old_content), ECContent)
# perform initial content creation
old_content.create(StringIO(data))
uploaded_content = self.content_factory.get(self.container_id,
old_content.content_id)
# break the content
old_info = {}
for pos in broken_pos_list:
old_info[pos] = {}
c = uploaded_content.chunks.filter(pos=pos)[0]
old_info[pos]["url"] = c.url
old_info[pos]["id"] = c.id
old_info[pos]["hash"] = c.checksum
chunk_id_to_rebuild = c.id
meta, stream = self.blob_client.chunk_get(c.url)
old_info[pos]["dl_meta"] = meta
old_info[pos]["dl_hash"] = md5_stream(stream)
# delete the chunk
self.blob_client.chunk_delete(c.url)
# rebuild the broken chunks
uploaded_content.rebuild_chunk(chunk_id_to_rebuild)
rebuilt_content = self.content_factory.get(self.container_id,
uploaded_content.content_id)
# sanity check
self.assertEqual(type(rebuilt_content), ECContent)
# verify rebuild result
for pos in broken_pos_list:
c = rebuilt_content.chunks.filter(pos=pos)[0]
rebuilt_meta, rebuilt_stream = self.blob_client.chunk_get(c.url)
self.assertEqual(rebuilt_meta["chunk_id"], c.id)
self.assertEqual(md5_stream(rebuilt_stream),
old_info[pos]["dl_hash"])
self.assertEqual(c.checksum, old_info[pos]["hash"])
self.assertNotEqual(c.url, old_info[pos]["url"])
del old_info[pos]["dl_meta"]["chunk_id"]
del rebuilt_meta["chunk_id"]
self.assertEqual(rebuilt_meta, old_info[pos]["dl_meta"])
def test_content_0_byte_rebuild(self):
self._test_rebuild(0, self.random_chunks(1))
def test_content_0_byte_rebuild_advanced(self):
self._test_rebuild(0, self.random_chunks(3))
def test_content_1_byte_rebuild(self):
self._test_rebuild(1, self.random_chunks(1))
def test_content_1_byte_rebuild_advanced(self):