Python ContainerClient.content_create方法代码示例

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class TestRebuilderCrawler(BaseTestCase):
    def setUp(self):
        super(TestRebuilderCrawler, self).setUp()

        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.gridconf = {"namespace": self.namespace}
        self.container_client = ContainerClient(self.gridconf)
        self.blob_client = BlobClient()

        self.container_name = "TestRebuilderCrawler%d" % int(time.time())
        self.container_client.container_create(acct=self.account,
                                               ref=self.container_name)

    def _push_content(self, content):
        for c in content.chunks:
            self.blob_client.chunk_put(c.url, c.get_create_xattr(), c.data)

        self.container_client.content_create(acct=content.account,
                                             ref=content.container_name,
                                             path=content.content_name,
                                             size=content.size,
                                             checksum=content.hash,
                                             content_id=content.content_id,
                                             stgpol=content.stgpol,
                                             data=content.get_create_meta2())

    def tearDown(self):
        super(TestRebuilderCrawler, self).tearDown()

    def test_rebuild_chunk(self):
        # push a new content
        content = TestContent(self.conf, self.account,
                              self.container_name, "mycontent", "TWOCOPIES")
        data = "azerty"
        content.add_chunk(data, pos='0', rawx=0)
        content.add_chunk(data, pos='0', rawx=1)

        self._push_content(content)

        # rebuild the first rawx
        rebuilder = BlobRebuilderWorker(self.gridconf, None,
                                        self.conf['rawx'][0]['addr'])

        rebuilder.chunk_rebuild(content.container_id, content.content_id,
                                content.chunks[0].id)

        # check meta2 information
        _, res = self.container_client.content_show(acct=content.account,
                                                    ref=content.container_name,
                                                    content=content.content_id)

        new_chunk_info = None
        for c in res:
            if (c['url'] != content.chunks[0].url and
                    c['url'] != content.chunks[1].url):
                new_chunk_info = c

        new_chunk_id = new_chunk_info['url'].split('/')[-1]

        self.assertEqual(new_chunk_info['hash'], content.chunks[0].hash)
        self.assertEqual(new_chunk_info['pos'], content.chunks[0].pos)
        self.assertEqual(new_chunk_info['size'], content.chunks[0].size)

        # check chunk information
        meta, stream = self.blob_client.chunk_get(new_chunk_info['url'])

        self.assertEqual(meta['content_size'], str(content.chunks[0].size))
        self.assertEqual(meta['content_path'], content.content_name)
        self.assertEqual(meta['content_cid'], content.container_id)
        self.assertEqual(meta['content_id'], content.content_id)
        self.assertEqual(meta['chunk_id'], new_chunk_id)
        self.assertEqual(meta['chunk_pos'], content.chunks[0].pos)
        self.assertEqual(meta['content_version'], content.version)
        self.assertEqual(meta['chunk_hash'], content.chunks[0].hash)

        self.assertEqual(stream.next(), content.chunks[0].data)

        # check rtime flag in rdir
        rdir_client = RdirClient(self.gridconf)
        res = rdir_client.chunk_fetch(self.conf['rawx'][0]['addr'])
        key = (content.container_id, content.content_id, content.chunks[0].id)
        for i_container, i_content, i_chunk, i_value in res:
            if (i_container, i_content, i_chunk) == key:
                check_value = i_value

        self.assertIsNotNone(check_value.get('rtime'))

    @unittest.skipIf(len(get_config()['rawx']) != 3,
                     "The number of rawx must be 3")
    def test_rebuild_no_spare(self):
        # push a new content
        content = TestContent(self.conf, self.account,
                              self.container_name, "mycontent", "THREECOPIES")
        data = "azerty"
        content.add_chunk(data, pos='0', rawx=0)
        content.add_chunk(data, pos='0', rawx=1)
        content.add_chunk(data, pos='0', rawx=2)

#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class TestBlobAuditorFunctional(BaseTestCase):
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.test_dir = self.conf['sds_path']

        self.chars = string.ascii_lowercase + string.ascii_uppercase +\
            string.digits
        self.chars_id = string.digits + 'ABCDEF'

        self.rawx = 'http://' + self.conf["rawx"][0]['addr']

        self.h = hashlib.new('md5')

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = rand_generator(self.chars, 8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = rand_generator(self.chars_id, 64)

        self.data = rand_generator(self.chars, 1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(
            rand_generator(self.chars, 6), len(self.data), self.url_rand, 1)

        self.content.id_container = cid_from_name(
            self.account, self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0,
                               self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
                            "size": self.chunk.size,
                            "url":  self.chunk_url}

        chunk_meta = {'content_size': self.content.size,
                      'content_chunksnb': self.content.nb_chunks,
                      'content_path': self.content.path,
                      'content_cid': self.content.id_container,
                      'content_id': '0000',
                      'content_version': 1,
                      'chunk_id': self.chunk.id_chunk,
                      'chunk_pos': self.chunk.pos}
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = self.test_dir + '/data/NS-rawx-1/' +\
            self.chunk.id_chunk[0:2] + "/" + self.chunk.id_chunk
        self.bad_container_id = '0'*64

    def tearDown(self):
        super(TestBlobAuditorFunctional, self).tearDown()

        try:
            self.container_c.content_delete(
                self.account, self.ref, self.content.path)
        except Exception:
            pass

        try:
            self.container_c.container_destroy(self.account, self.ref)
        except Exception:
            pass

        try:
            os.remove(self.chunk_path)
        except Exception:
            pass

    def init_content(self):
        self.container_c.content_create(
            self.account, self.ref, self.content.path, self.chunk.size,
            self.hash_rand, data=[self.chunk_proxy])

    def test_chunk_audit(self):
        self.init_content()
        self.auditor.chunk_audit(self.chunk_path)

    def test_content_deleted(self):
        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
                          self.chunk_path)

    def test_container_deleted(self):
        self.container_c.container_destroy(self.account, self.ref)

        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
                          self.chunk_path)

    def test_chunk_corrupted(self):
        self.init_content()
        with open(self.chunk_path, "w") as f:
            f.write(rand_generator(self.chars, 1280))
#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class Content(object):
    def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
        self.conf = conf
        self.container_id = container_id
        self.metadata = metadata
        self.chunks = ChunksHelper(chunks)
        self.stgpol_args = stgpol_args
        self.logger = get_logger(self.conf)
        self.cs_client = ConscienceClient(conf)
        self.container_client = ContainerClient(self.conf)
        self.blob_client = BlobClient()
        self.session = requests.Session()
        self.content_id = self.metadata["id"]
        self.stgpol_name = self.metadata["policy"]
        self.path = self.metadata["name"]
        self.length = int(self.metadata["length"])
        self.version = self.metadata["version"]
        self.hash = self.metadata["hash"]
        self.mime_type = self.metadata["mime-type"]
        self.chunk_method = self.metadata["chunk-method"]

    def _meta2_get_spare_chunk(self, chunks_notin, chunks_broken):
        spare_data = {
            "notin": ChunksHelper(chunks_notin, False).raw(),
            "broken": ChunksHelper(chunks_broken, False).raw()
        }
        try:
            spare_resp = self.container_client.content_spare(
                cid=self.container_id, content=self.content_id,
                data=spare_data, stgpol=self.stgpol_name)
        except ClientException as e:
            raise exc.SpareChunkException("No spare chunk (%s)" % e.message)

        url_list = []
        for c in spare_resp["chunks"]:
            url_list.append(c["id"])

        return url_list

    def _meta2_update_spare_chunk(self, current_chunk, new_url):
        old = [{'type': 'chunk',
                'id': current_chunk.url,
                'hash': current_chunk.hash,
                'size': current_chunk.size,
                'pos': current_chunk.pos,
                'content': self.content_id}]
        new = [{'type': 'chunk',
                'id': new_url,
                'hash': current_chunk.hash,
                'size': current_chunk.size,
                'pos': current_chunk.pos,
                'content': self.content_id}]
        update_data = {'old': old, 'new': new}

        self.container_client.container_raw_update(
            cid=self.container_id, data=update_data)

    def _meta2_create_object(self):
        self.container_client.content_create(cid=self.container_id,
                                             path=self.path,
                                             content_id=self.content_id,
                                             stgpol=self.stgpol_name,
                                             size=self.length,
                                             checksum=self.hash,
                                             version=self.version,
                                             chunk_method=self.chunk_method,
                                             mime_type=self.mime_type,
                                             data=self.chunks.raw())

    def rebuild_chunk(self, chunk_id):
        raise NotImplementedError()

    def upload(self, stream):
        try:
            self._upload(stream)
        except Exception as e:
            for chunk in self.chunks:
                try:
                    self.blob_client.chunk_delete(chunk.url)
                except:
                    pass
            raise e

    def _upload(self, stream):
        raise NotImplementedError()

    def download(self):
        raise NotImplementedError()

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class Content(object):
    def __init__(self, conf, container_id, metadata, chunks, stgpol_args):
        self.conf = conf
        self.container_id = container_id
        self.metadata = metadata
        self.chunks = ChunksHelper(chunks)
        self.stgpol_args = stgpol_args
        self.logger = get_logger(self.conf)
        self.cs_client = ConscienceClient(conf)
        self.container_client = ContainerClient(self.conf)
        self.blob_client = BlobClient()
        self.session = requests.Session()
        self.content_id = self.metadata["id"]
        self.stgpol_name = self.metadata["policy"]
        self.path = self.metadata["name"]
        self.length = int(self.metadata["length"])
        self.version = self.metadata["version"]
        self.hash = self.metadata["hash"]
        self.mime_type = self.metadata["mime-type"]
        self.chunk_method = self.metadata["chunk-method"]

    def _meta2_get_spare_chunk(self, chunks_notin, chunks_broken):
        spare_data = {
            "notin": ChunksHelper(chunks_notin, False).raw(),
            "broken": ChunksHelper(chunks_broken, False).raw()
        }
        try:
            spare_resp = self.container_client.content_spare(
                cid=self.container_id, content=self.content_id,
                data=spare_data, stgpol=self.stgpol_name)
        except ClientException as e:
            raise exc.SpareChunkException("No spare chunk (%s)" % e.message)

        url_list = []
        for c in spare_resp["chunks"]:
            url_list.append(c["id"])

        return url_list

    def _meta2_update_spare_chunk(self, current_chunk, new_url):
        old = [{'type': 'chunk',
                'id': current_chunk.url,
                'hash': current_chunk.hash,
                'size': current_chunk.size,
                'pos': current_chunk.pos,
                'content': self.content_id}]
        new = [{'type': 'chunk',
                'id': new_url,
                'hash': current_chunk.hash,
                'size': current_chunk.size,
                'pos': current_chunk.pos,
                'content': self.content_id}]
        update_data = {'old': old, 'new': new}

        self.container_client.container_raw_update(
            cid=self.container_id, data=update_data)

    def _meta2_create_object(self):
        self.container_client.content_create(cid=self.container_id,
                                             path=self.path,
                                             content_id=self.content_id,
                                             stgpol=self.stgpol_name,
                                             size=self.length,
                                             checksum=self.hash,
                                             version=self.version,
                                             chunk_method=self.chunk_method,
                                             mime_type=self.mime_type,
                                             data=self.chunks.raw())

    def rebuild_chunk(self, chunk_id):
        raise NotImplementedError()

    def upload(self, stream):
        try:
            self._upload(stream)
        except:
            # Keep the stack trace
            exc_info = sys.exc_info()
            for chunk in self.chunks:
                try:
                    self.blob_client.chunk_delete(chunk.url)
                except:
                    self.logger.warn("Failed to delete %s", chunk.url)
            # Raise with the original stack trace
            raise exc_info[0], exc_info[1], exc_info[2]

    def _upload(self, stream):
        raise NotImplementedError()

    def download(self):
        raise NotImplementedError()

    def delete(self):
        self.container_client.content_delete(cid=self.container_id,
                                             path=self.path)

    def move_chunk(self, chunk_id):
        current_chunk = self.chunks.filter(id=chunk_id).one()
        if current_chunk is None:
            raise OrphanChunk("Chunk not found in content")
#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class TestBlobAuditorFunctional(BaseTestCase):
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.test_dir = self.conf['sds_path']

        rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
        self.rawx = 'http://' + rawx_addr

        self.h = hashlib.new('md5')

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = random_str(8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = random_id(64)

        self.data = random_str(1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(
            random_str(6), len(self.data), self.url_rand, 1)

        self.content.id_container = cid_from_name(
            self.account, self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0,
                               self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
                            "size": self.chunk.size,
                            "url":  self.chunk_url}

        chunk_meta = {'content_path': self.content.path,
                      'container_id': self.content.id_container,
                      'content_chunkmethod': 'plain/nb_copy=3',
                      'content_policy': 'TESTPOLICY',
                      'content_id': '0000',
                      'content_version': 1,
                      'chunk_id': self.chunk.id_chunk,
                      'chunk_pos': self.chunk.pos}
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = self.test_dir + '/data/' + self.namespace + \
            '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
        self.bad_container_id = '0'*64

    def tearDown(self):
        super(TestBlobAuditorFunctional, self).tearDown()

        try:
            self.container_c.content_delete(
                self.account, self.ref, self.content.path)
        except Exception:
            pass

        try:
            self.container_c.container_destroy(self.account, self.ref)
        except Exception:
            pass

        try:
            os.remove(self.chunk_path)
        except Exception:
            pass

    def init_content(self):
        self.container_c.content_create(
            self.account, self.ref, self.content.path, self.chunk.size,
            self.hash_rand, data=[self.chunk_proxy])

    def test_chunk_audit(self):
        self.init_content()
        self.auditor.chunk_audit(self.chunk_path)

    def test_content_deleted(self):
        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
                          self.chunk_path)

    def test_container_deleted(self):
        self.container_c.container_destroy(self.account, self.ref)

        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit,
                          self.chunk_path)

    def test_chunk_corrupted(self):
        self.init_content()
        with open(self.chunk_path, "w") as f:
            f.write(random_str(1280))

        self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
                          self.chunk_path)
#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_create [as 别名]
class TestBlobAuditorFunctional(BaseTestCase):
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf["namespace"]
        self.account = self.conf["account"]

        self.test_dir = self.conf["sds_path"]

        rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
        self.rawx = "http://" + rawx_addr

        self.h = hashlib.new("md5")

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = random_str(8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = random_id(64)

        self.data = random_str(1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)

        self.content.id_container = cid_from_name(self.account, self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}

        chunk_meta = {
            "content_path": self.content.path,
            "container_id": self.content.id_container,
            "chunk_method": "plain/nb_copy=3",
            "policy": "TESTPOLICY",
            "id": "0000",
            "version": 1,
            "chunk_id": self.chunk.id_chunk,
            "chunk_pos": self.chunk.pos,
            "chunk_hash": self.chunk.md5,
        }
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = (
            self.test_dir
            + "/data/"
            + self.namespace
            + "-rawx-1/"
            + self.chunk.id_chunk[0:3]
            + "/"
            + self.chunk.id_chunk
        )
        self.bad_container_id = "0" * 64

    def tearDown(self):
        super(TestBlobAuditorFunctional, self).tearDown()

        try:
            self.container_c.content_delete(self.account, self.ref, self.content.path)
        except Exception:
            pass

        try:
            self.container_c.container_destroy(self.account, self.ref)
        except Exception:
            pass

        try:
            os.remove(self.chunk_path)
        except Exception:
            pass

    def init_content(self):
        self.container_c.content_create(
            self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]
        )

    def test_chunk_audit(self):
        self.init_content()
        self.auditor.chunk_audit(self.chunk_path)

    def test_content_deleted(self):
        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)

    def test_container_deleted(self):
        self.container_c.container_destroy(self.account, self.ref)

        self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)

    def test_chunk_corrupted(self):
        self.init_content()
        with open(self.chunk_path, "w") as f:
            f.write(random_str(1280))

#.........这里部分代码省略.........