Python ContainerClient.content_show方法代码示例

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class ContentFactory(object):
    DEFAULT_DATASEC = "plain", {"nb_copy": "1", "distance": "0"}

    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf)
        self.container_client = ContainerClient(conf)

    def get(self, container_id, content_id):
        try:
            meta, chunks = self.container_client.content_show(
                cid=container_id, content=content_id)
        except NotFound:
            raise ContentNotFound("Content %s/%s not found" % (container_id,
                                  content_id))

        chunk_method = meta['chunk_method']
        storage_method = STORAGE_METHODS.load(chunk_method)

        cls = ECContent if storage_method.ec else PlainContent
        return cls(self.conf, container_id, meta, chunks, storage_method)

    def new(self, container_id, path, size, policy):
        meta, chunks = self.container_client.content_prepare(
            cid=container_id, path=path, size=size, stgpol=policy)

        chunk_method = meta['chunk_method']
        storage_method = STORAGE_METHODS.load(chunk_method)

        cls = ECContent if storage_method.ec else PlainContent
        return cls(self.conf, container_id, meta, chunks, storage_method)

    def change_policy(self, container_id, content_id, new_policy):
        old_content = self.get(container_id, content_id)
        if old_content.stgpol == new_policy:
            return old_content

        new_content = self.new(container_id, old_content.path,
                               old_content.length, new_policy)

        stream = old_content.fetch()
        new_content.create(GeneratorIO(stream))
        # the old content is automatically deleted because the new content has
        # the same name (but not the same id)
        return new_content

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]

#.........这里部分代码省略.........
                self.errors = 0
                self.bytes_processed = 0
                self.last_reported = now
            audit_time += (now - loop_time)
        elapsed = (time.time() - start_time) or 0.000001
        self.logger.info(
            '%(elapsed).02f '
            '%(corrupted)d '
            '%(faulty)d '
            '%(orphans)d '
            '%(errors)d '
            '%(chunk_rate).2f '
            '%(bytes_rate).2f '
            '%(audit_time).2f '
            '%(audit_rate).2f' % {
                'elapsed': elapsed,
                'corrupted': total_corrupted + self.corrupted_chunks,
                'faulty': total_faulty + self.faulty_chunks,
                'orphans': total_orphans + self.orphan_chunks,
                'errors': total_errors + self.errors,
                'chunk_rate': self.total_chunks_processed / elapsed,
                'bytes_rate': self.total_bytes_processed / elapsed,
                'audit_time': audit_time,
                'audit_rate': audit_time / elapsed
            }
        )

    def safe_chunk_audit(self, path):
        try:
            self.chunk_audit(path)
        except exc.FaultyChunk as err:
            self.faulty_chunks += 1
            self.logger.error('ERROR faulty chunk %s: %s', path, err)
        except exc.CorruptedChunk as err:
            self.corrupted_chunks += 1
            self.logger.error('ERROR corrupted chunk %s: %s', path, err)
        except exc.OrphanChunk as err:
            self.orphan_chunks += 1
            self.logger.error('ERROR orphan chunk %s: %s', path, err)
        except Exception:
            self.errors += 1
            self.logger.exception('ERROR while auditing chunk %s', path)

        self.passes += 1

    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk(
                    'Missing extended attribute %s' % e)
            size = int(meta['chunk_size'])
            md5_checksum = meta['chunk_hash'].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        increment=buf_len)
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                content_cid = meta['content_cid']
                content_path = meta['content_path']
                content_attr, data = self.container_client.content_show(
                    cid=content_cid, path=content_path)

                # Check chunk data
                chunk_data = None
                metachunks = set()
                for c in data:
                    if c['url'].endswith(meta['chunk_id']):
                        metachunks.add(c['pos'].split('.', 2)[0])
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk('Not found in content')

                if chunk_data['size'] != int(meta['chunk_size']):
                    raise exc.FaultyChunk('Invalid chunk size found')

                if chunk_data['hash'] != meta['chunk_hash']:
                    raise exc.FaultyChunk('Invalid chunk hash found')

                if chunk_data['pos'] != meta['chunk_pos']:
                    raise exc.FaultyChunk('Invalid chunk position found')

                # Check content data
                if content_attr['length'] != meta['content_size']:
                    raise exc.FaultyChunk('Invalid content size found')

                if len(metachunks) != int(meta['content_chunksnb']):
                    self.logger.warn('Invalid number of chunks found')
                    raise exc.FaultyChunk('Invalid number of chunks found')

            except exc.NotFound:
                raise exc.OrphanChunk('Chunk not found in container')

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]

#.........这里部分代码省略.........
                )
                report_time = now
                total_corrupted += self.corrupted_chunks
                total_orphans += self.orphan_chunks
                total_faulty += self.faulty_chunks
                total_errors += self.errors
                self.passes = 0
                self.corrupted_chunks = 0
                self.orphan_chunks = 0
                self.faulty_chunks = 0
                self.errors = 0
                self.bytes_processed = 0
                self.last_reported = now
            audit_time += now - loop_time
        elapsed = (time.time() - start_time) or 0.000001
        self.logger.info(
            "%(elapsed).02f "
            "%(corrupted)d "
            "%(faulty)d "
            "%(orphans)d "
            "%(errors)d "
            "%(chunk_rate).2f "
            "%(bytes_rate).2f "
            "%(audit_time).2f "
            "%(audit_rate).2f"
            % {
                "elapsed": elapsed,
                "corrupted": total_corrupted + self.corrupted_chunks,
                "faulty": total_faulty + self.faulty_chunks,
                "orphans": total_orphans + self.orphan_chunks,
                "errors": total_errors + self.errors,
                "chunk_rate": self.total_chunks_processed / elapsed,
                "bytes_rate": self.total_bytes_processed / elapsed,
                "audit_time": audit_time,
                "audit_rate": audit_time / elapsed,
            }
        )

    def safe_chunk_audit(self, path):
        try:
            self.chunk_audit(path)
        except exc.FaultyChunk as err:
            self.faulty_chunks += 1
            self.logger.error("ERROR faulty chunk %s: %s", path, err)
        except exc.CorruptedChunk as err:
            self.corrupted_chunks += 1
            self.logger.error("ERROR corrupted chunk %s: %s", path, err)
        except exc.OrphanChunk as err:
            self.orphan_chunks += 1
            self.logger.error("ERROR orphan chunk %s: %s", path, err)
        except Exception:
            self.errors += 1
            self.logger.exception("ERROR while auditing chunk %s", path)

        self.passes += 1

    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk("Missing extended attribute %s" % e)
            size = int(meta["chunk_size"])
            md5_checksum = meta["chunk_hash"].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time, self.max_bytes_per_second, increment=buf_len
                    )
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                container_id = meta["container_id"]
                content_path = meta["content_path"]
                content_attr, data = self.container_client.content_show(cid=container_id, path=content_path)

                # Check chunk data
                chunk_data = None
                metachunks = set()
                for c in data:
                    if c["url"].endswith(meta["chunk_id"]):
                        metachunks.add(c["pos"].split(".", 2)[0])
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk("Not found in content")

                if chunk_data["size"] != int(meta["chunk_size"]):
                    raise exc.FaultyChunk("Invalid chunk size found")

                if chunk_data["hash"] != meta["chunk_hash"]:
                    raise exc.FaultyChunk("Invalid chunk hash found")

                if chunk_data["pos"] != meta["chunk_pos"]:
                    raise exc.FaultyChunk("Invalid chunk position found")

            except exc.NotFound:
                raise exc.OrphanChunk("Chunk not found in container")

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]

#.........这里部分代码省略.........
                    '%(errors)d '
                    '%(c_rate).2f '
                    '%(b_rate).2f '
                    '%(total).2f '
                    '%(mover_time).2f'
                    '%(mover_rate).2f' % {
                        'start_time': time.ctime(report_time),
                        'passes': self.passes,
                        'errors': self.errors,
                        'c_rate': self.passes / (now - report_time),
                        'b_rate': self.bytes_processed / (now - report_time),
                        'total': (now - start_time),
                        'mover_time': mover_time,
                        'mover_rate': mover_time / (now - start_time)
                    }
                )
                report_time = now
                total_errors += self.errors
                self.passes = 0
                self.bytes_processed = 0
                self.last_reported = now
            mover_time += (now - loop_time)
        elapsed = (time.time() - start_time) or 0.000001
        self.logger.info(
            '%(elapsed).02f '
            '%(errors)d '
            '%(chunk_rate).2f '
            '%(bytes_rate).2f '
            '%(mover_time).2f '
            '%(mover_rate).2f' % {
                'elapsed': elapsed,
                'errors': total_errors + self.errors,
                'chunk_rate': self.total_chunks_processed / elapsed,
                'bytes_rate': self.total_bytes_processed / elapsed,
                'mover_time': mover_time,
                'mover_rate': mover_time / elapsed
            }
        )

    def safe_chunk_move(self, path):
        try:
            self.chunk_move(path)
        except Exception as e:
            self.errors += 1
            self.logger.error('ERROR while moving chunk %s: %s', path, e)
        self.passes += 1

    def load_chunk_metadata(self, path):
        with open(path) as f:
            return read_chunk_metadata(f)

    def chunk_move(self, path):
        meta = self.load_chunk_metadata(path)
        content_cid = meta['content_cid']
        content_path = meta['content_path']

        chunk_url = 'http://%s/%s' % \
            (self.address, meta['chunk_id'])

        try:
            data = self.container_client.content_show(
                cid=content_cid, path=content_path)
        except exc.NotFound:
            raise exc.OrphanChunk('Content not found')
        current_chunk = None
        notin = []
        for c in data:
            if c['pos'] == meta['chunk_pos']:
                notin.append(c)
        for c in notin:
            if c['url'] == chunk_url:
                current_chunk = c
                notin.remove(c)
        if not current_chunk:
            raise exc.OrphanChunk('Chunk not found in content')
        spare_data = {'notin': notin, 'broken': [current_chunk], 'size': 0}
        spare_resp = self.container_client.content_spare(
            cid=content_cid, path=content_path, data=spare_data)

        new_chunk = spare_resp['chunks'][0]
        self.blob_client.chunk_copy(
            current_chunk['url'], new_chunk['id'])

        old = [{'type': 'chunk',
                'id': current_chunk['url'],
                'hash': meta['chunk_hash'],
                'size': int(meta['chunk_size'])}]
        new = [{'type': 'chunk',
                'id': new_chunk['id'],
                'hash': meta['chunk_hash'],
                'size': int(meta['chunk_size'])}]
        update_data = {'old': old, 'new': new}

        self.container_client.container_raw_update(
            cid=content_cid, data=update_data)

        self.blob_client.chunk_delete(current_chunk['url'])

        self.logger.info(
            'moved chunk %s to %s', current_chunk['url'], new_chunk['id'])

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class TestDupContent(BaseTestCase):
    def setUp(self):
        super(TestDupContent, self).setUp()

        if len(self.conf['rawx']) < 3:
            self.skipTest("Not enough rawx. "
                          "Dup tests needs more than 2 rawx to run")

        self.namespace = self.conf['namespace']
        self.account = self.conf['account']
        self.chunk_size = self.conf['chunk_size']
        self.gridconf = {"namespace": self.namespace}
        self.content_factory = ContentFactory(self.gridconf)
        self.container_client = ContainerClient(self.gridconf)
        self.blob_client = BlobClient()
        self.container_name = "TestDupContent%f" % time.time()
        self.container_client.container_create(acct=self.account,
                                               ref=self.container_name)
        self.container_id = cid_from_name(self.account,
                                          self.container_name).upper()

    def tearDown(self):
        super(TestDupContent, self).tearDown()

    def _test_upload(self, stgpol, data_size):
        data = random_data(data_size)
        content = self.content_factory.new(self.container_id, "titi",
                                           len(data), stgpol)
        self.assertEqual(type(content), DupContent)

        content.upload(StringIO.StringIO(data))

        meta, chunks = self.container_client.content_show(
            cid=self.container_id, content=content.content_id)
        chunks = ChunksHelper(chunks)
        self.assertEqual(meta['hash'], md5_data(data))
        self.assertEqual(meta['length'], str(len(data)))
        self.assertEqual(meta['policy'], stgpol)
        self.assertEqual(meta['name'], "titi")

        metachunk_nb = int(math.ceil(float(len(data)) / self.chunk_size))
        if metachunk_nb == 0:
            metachunk_nb = 1  # special case for empty content

        if stgpol == "THREECOPIES":
            nb_copy = 3
        elif stgpol == "TWOCOPIES":
            nb_copy = 2
        elif stgpol == "SINGLE":
            nb_copy = 1

        self.assertEqual(len(chunks), metachunk_nb * nb_copy)

        for pos in range(metachunk_nb):
            chunks_at_pos = chunks.filter(pos=pos)
            self.assertEqual(len(chunks_at_pos), nb_copy)

            data_begin = pos * self.chunk_size
            data_end = pos * self.chunk_size + self.chunk_size
            chunk_hash = md5_data(data[data_begin:data_end])

            for chunk in chunks_at_pos:
                meta, stream = self.blob_client.chunk_get(chunk.url)
                self.assertEqual(md5_stream(stream), chunk_hash)
                self.assertEqual(meta['content_size'], str(len(data)))
                self.assertEqual(meta['content_path'], "titi")
                self.assertEqual(meta['content_cid'], self.container_id)
                self.assertEqual(meta['content_id'], meta['content_id'])
                self.assertEqual(meta['chunk_id'], chunk.id)
                self.assertEqual(meta['chunk_pos'], str(pos))
                self.assertEqual(meta['chunk_hash'], chunk_hash)

    def test_twocopies_upload_0_byte(self):
        self._test_upload("TWOCOPIES", 0)

    def test_twocopies_upload_1_byte(self):
        self._test_upload("TWOCOPIES", 1)

    def test_twocopies_upload_chunksize_bytes(self):
        self._test_upload("TWOCOPIES", self.chunk_size)

    def test_twocopies_upload_chunksize_plus_1_bytes(self):
        self._test_upload("TWOCOPIES", self.chunk_size + 1)

    def test_single_upload_0_byte(self):
        self._test_upload("SINGLE", 0)

    def test_single_upload_chunksize_plus_1_bytes(self):
        self._test_upload("SINGLE", self.chunk_size + 1)

    def test_chunks_cleanup_when_upload_failed(self):
        data = random_data(2 * self.chunk_size)
        content = self.content_factory.new(self.container_id, "titi",
                                           len(data), "TWOCOPIES")
        self.assertEqual(type(content), DupContent)

        # set bad url for position 1
        for chunk in content.chunks.filter(pos=1):
            chunk.url = "http://127.0.0.1:9/DEADBEEF"

#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class TestRebuilderCrawler(BaseTestCase):
    def setUp(self):
        super(TestRebuilderCrawler, self).setUp()

        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.gridconf = {"namespace": self.namespace}
        self.container_client = ContainerClient(self.gridconf)
        self.blob_client = BlobClient()

        self.container_name = "TestRebuilderCrawler%d" % int(time.time())
        self.container_client.container_create(acct=self.account,
                                               ref=self.container_name)

    def _push_content(self, content):
        for c in content.chunks:
            self.blob_client.chunk_put(c.url, c.get_create_xattr(), c.data)

        self.container_client.content_create(acct=content.account,
                                             ref=content.container_name,
                                             path=content.content_name,
                                             size=content.size,
                                             checksum=content.hash,
                                             content_id=content.content_id,
                                             stgpol=content.stgpol,
                                             data=content.get_create_meta2())

    def tearDown(self):
        super(TestRebuilderCrawler, self).tearDown()

    def test_rebuild_chunk(self):
        # push a new content
        content = TestContent(self.conf, self.account,
                              self.container_name, "mycontent", "TWOCOPIES")
        data = "azerty"
        content.add_chunk(data, pos='0', rawx=0)
        content.add_chunk(data, pos='0', rawx=1)

        self._push_content(content)

        # rebuild the first rawx
        rebuilder = BlobRebuilderWorker(self.gridconf, None,
                                        self.conf['rawx'][0]['addr'])

        rebuilder.chunk_rebuild(content.container_id, content.content_id,
                                content.chunks[0].id)

        # check meta2 information
        _, res = self.container_client.content_show(acct=content.account,
                                                    ref=content.container_name,
                                                    content=content.content_id)

        new_chunk_info = None
        for c in res:
            if (c['url'] != content.chunks[0].url and
                    c['url'] != content.chunks[1].url):
                new_chunk_info = c

        new_chunk_id = new_chunk_info['url'].split('/')[-1]

        self.assertEqual(new_chunk_info['hash'], content.chunks[0].hash)
        self.assertEqual(new_chunk_info['pos'], content.chunks[0].pos)
        self.assertEqual(new_chunk_info['size'], content.chunks[0].size)

        # check chunk information
        meta, stream = self.blob_client.chunk_get(new_chunk_info['url'])

        self.assertEqual(meta['content_size'], str(content.chunks[0].size))
        self.assertEqual(meta['content_path'], content.content_name)
        self.assertEqual(meta['content_cid'], content.container_id)
        self.assertEqual(meta['content_id'], content.content_id)
        self.assertEqual(meta['chunk_id'], new_chunk_id)
        self.assertEqual(meta['chunk_pos'], content.chunks[0].pos)
        self.assertEqual(meta['content_version'], content.version)
        self.assertEqual(meta['chunk_hash'], content.chunks[0].hash)

        self.assertEqual(stream.next(), content.chunks[0].data)

        # check rtime flag in rdir
        rdir_client = RdirClient(self.gridconf)
        res = rdir_client.chunk_fetch(self.conf['rawx'][0]['addr'])
        key = (content.container_id, content.content_id, content.chunks[0].id)
        for i_container, i_content, i_chunk, i_value in res:
            if (i_container, i_content, i_chunk) == key:
                check_value = i_value

        self.assertIsNotNone(check_value.get('rtime'))

    @unittest.skipIf(len(get_config()['rawx']) != 3,
                     "The number of rawx must be 3")
    def test_rebuild_no_spare(self):
        # push a new content
        content = TestContent(self.conf, self.account,
                              self.container_name, "mycontent", "THREECOPIES")
        data = "azerty"
        content.add_chunk(data, pos='0', rawx=0)
        content.add_chunk(data, pos='0', rawx=1)
        content.add_chunk(data, pos='0', rawx=2)

#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class TestRainContent(BaseTestCase):
    def setUp(self):
        super(TestRainContent, self).setUp()

        if len(self.conf['rawx']) < 12:
            self.skipTest("Not enough rawx. "
                          "Rain tests needs more than 12 rawx to run")

        self.namespace = self.conf['namespace']
        self.account = self.conf['account']
        self.chunk_size = self.conf['chunk_size']
        self.gridconf = {"namespace": self.namespace}
        self.content_factory = ContentFactory(self.gridconf)
        self.container_client = ContainerClient(self.gridconf)
        self.blob_client = BlobClient()
        self.container_name = "TestRainContent%f" % time.time()
        self.container_client.container_create(acct=self.account,
                                               ref=self.container_name)
        self.container_id = cid_from_name(self.account,
                                          self.container_name).upper()

    def tearDown(self):
        super(TestRainContent, self).tearDown()

    def _test_upload(self, data_size):
        data = random_data(data_size)
        content = self.content_factory.new(self.container_id, "titi",
                                           len(data), "RAIN")
        k = 6
        m = 2
        self.assertEqual(type(content), RainContent)

        content.upload(StringIO.StringIO(data))

        meta, chunks = self.container_client.content_show(
            cid=self.container_id, content=content.content_id)
        chunks = ChunksHelper(chunks)
        self.assertEqual(meta['hash'], md5_data(data))
        self.assertEqual(meta['length'], str(len(data)))
        self.assertEqual(meta['policy'], "RAIN")
        self.assertEqual(meta['name'], "titi")

        metachunk_nb = int(math.ceil(float(len(data)) / self.chunk_size))
        if metachunk_nb == 0:
            metachunk_nb = 1  # special case for empty content

        nb_chunks_min = metachunk_nb * (1 + m)
        nb_chunks_max = metachunk_nb * (k + m)
        self.assertGreaterEqual(len(chunks), nb_chunks_min)
        self.assertLessEqual(len(chunks), nb_chunks_max)

        for metapos in range(metachunk_nb):
            chunks_at_pos = content.chunks.filter(metapos=metapos)
            data_chunks_at_pos = chunks_at_pos.filter(is_parity=False)
            parity_chunks_at_pos = chunks_at_pos.filter(is_parity=True)

            self.assertEquals(len(data_chunks_at_pos) >= 1, True)
            self.assertEquals(len(data_chunks_at_pos) <= k, True)
            self.assertEqual(len(parity_chunks_at_pos), m)

            for chunk in chunks_at_pos:
                meta, stream = self.blob_client.chunk_get(chunk.url)
                self.assertEqual(md5_stream(stream), chunk.hash)
                self.assertEqual(meta['content_size'], str(len(data)))
                self.assertEqual(meta['content_path'], "titi")
                self.assertEqual(meta['content_cid'], self.container_id)
                self.assertEqual(meta['content_id'], meta['content_id'])
                self.assertEqual(meta['chunk_id'], chunk.id)
                self.assertEqual(meta['chunk_pos'], chunk.pos)
                self.assertEqual(meta['chunk_hash'], chunk.hash)

            data_begin = metapos * self.chunk_size
            data_end = metapos * self.chunk_size + self.chunk_size
            target_metachunk_hash = md5_data(data[data_begin:data_end])

            metachunk_hash = hashlib.md5()
            for chunk in data_chunks_at_pos:
                meta, stream = self.blob_client.chunk_get(chunk.url)
                for d in stream:
                    metachunk_hash.update(d)
            self.assertEqual(metachunk_hash.hexdigest().upper(),
                             target_metachunk_hash)

    def test_upload_0_byte(self):
        self._test_upload(0)

    def test_upload_1_byte(self):
        self._test_upload(1)

    def test_upload_chunksize_bytes(self):
        self._test_upload(self.chunk_size)

    def test_upload_chunksize_plus_1_bytes(self):
        self._test_upload(self.chunk_size + 1)

    def test_chunks_cleanup_when_upload_failed(self):
        data = random_data(2 * self.chunk_size)
        content = self.content_factory.new(self.container_id, "titi",
                                           len(data), "RAIN")
        self.assertEqual(type(content), RainContent)
#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]

#.........这里部分代码省略.........
            '%(rebuilder_time).2f '
            '%(rebuilder_rate).2f' % {
                'elapsed': elapsed,
                'errors': total_errors + self.errors,
                'chunk_rate': self.total_chunks_processed / elapsed,
                'bytes_rate': self.total_bytes_processed / elapsed,
                'rebuilder_time': rebuilder_time,
                'rebuilder_rate': rebuilder_time / elapsed
            }
        )

    def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id):
        self.logger.info("[dryrun] Rebuilding "
                         "container %s, content %s, chunk %s"
                         % (container_id, content_id, chunk_id))
        self.passes += 1

    def safe_chunk_rebuild(self, container_id, content_id, chunk_id):
        self.logger.info('Rebuilding (container %s, content %s, chunk %s)'
                         % (container_id, content_id, chunk_id))
        try:
            self.chunk_rebuild(container_id, content_id, chunk_id)
        except Exception as e:
            self.errors += 1
            self.logger.error('ERROR while rebuilding chunk %s|%s|%s) : %s',
                              container_id, content_id, chunk_id, e)

        self.passes += 1

    def _meta2_get_chunks_at_pos(self, container_id, content_id, chunk_id):
        current_chunk_url = 'http://%s/%s' % (self.volume, chunk_id)

        try:
            data = self.container_client.content_show(
                cid=container_id, content=content_id)
        except exc.NotFound:
            raise exc.OrphanChunk('Content not found')

        current_chunk = None
        for c in data:
            if c['url'] == current_chunk_url:
                current_chunk = c
                break
        if not current_chunk:
            raise exc.OrphanChunk('Chunk not found in content')

        duplicate_chunks = []
        for c in data:
            if c['pos'] == current_chunk['pos'] \
                    and c['url'] != current_chunk['url']:
                duplicate_chunks.append(c)
        if len(duplicate_chunks) == 0:
            raise exc.UnrecoverableContent('No copy of missing chunk')

        return current_chunk, duplicate_chunks

    def _meta2_get_spare_chunk(self, container_id, content_id, notin, broken):
        spare_data = {'notin': notin,
                      'broken': [broken],
                      'size': 0}
        try:
            spare_resp = self.container_client.content_spare(
                cid=container_id, content=content_id, data=spare_data)
        except ClientException as e:
            raise exc.SpareChunkException('No spare chunk (%s)' % e.message)

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class TestECContent(BaseTestCase):
    def setUp(self):
        super(TestECContent, self).setUp()

        if len(self.conf['services']['rawx']) < 12:
            self.skipTest("Not enough rawx. "
                          "EC tests needs at least 12 rawx to run")

        self.namespace = self.conf['namespace']
        self.account = self.conf['account']
        self.chunk_size = self.conf['chunk_size']
        self.gridconf = {"namespace": self.namespace}
        self.content_factory = ContentFactory(self.gridconf)
        self.container_client = ContainerClient(self.gridconf)
        self.blob_client = BlobClient()
        self.container_name = "TestECContent%f" % time.time()
        self.container_client.container_create(acct=self.account,
                                               ref=self.container_name)
        self.container_id = cid_from_name(self.account,
                                          self.container_name).upper()
        self.content = random_str(64)
        self.stgpol = "EC"
        self.size = 1024*1024 + 320
        self.k = 6
        self.m = 3

    def tearDown(self):
        super(TestECContent, self).tearDown()

    def random_chunks(self, nb):
        l = random.sample(xrange(self.k + self.m), nb)
        return ["0.%s" % i for i in l]

    def _test_create(self, data_size):
        # generate random test data
        data = random_data(data_size)
        # using factory create new EC content
        content = self.content_factory.new(
            self.container_id, self.content, len(data), self.stgpol)
        # verify the factory gave us an ECContent
        self.assertEqual(type(content), ECContent)

        # perform the content creation
        content.create(StringIO(data))

        meta, chunks = self.container_client.content_show(
            cid=self.container_id, content=content.content_id)
        # verify metadata
        chunks = ChunksHelper(chunks)
        self.assertEqual(meta['hash'], md5_data(data))
        self.assertEqual(meta['length'], str(len(data)))
        self.assertEqual(meta['policy'], self.stgpol)
        self.assertEqual(meta['name'], self.content)

        metachunk_nb = int(math.ceil(float(len(data)) / self.chunk_size)) \
            if len(data) != 0 else 1

        # verify each metachunk
        for metapos in range(metachunk_nb):
            chunks_at_pos = content.chunks.filter(metapos=metapos)

            for chunk in chunks_at_pos:
                meta, stream = self.blob_client.chunk_get(chunk.url)
                self.assertEqual(meta['metachunk_size'], str(chunk.size))
                self.assertEqual(meta['metachunk_hash'], chunk.checksum)
                self.assertEqual(meta['content_path'], self.content)
                self.assertEqual(meta['container_id'], self.container_id)
                self.assertEqual(meta['content_id'], meta['content_id'])
                self.assertEqual(meta['chunk_id'], chunk.id)
                self.assertEqual(meta['chunk_pos'], chunk.pos)
                self.assertEqual(meta['chunk_hash'], md5_stream(stream))

    def test_create_0_byte(self):
        self._test_create(0)

    def test_create_1_byte(self):
        self._test_create(1)

    def test_create(self):
        self._test_create(DAT_LEGIT_SIZE)

    def _test_rebuild(self, data_size, broken_pos_list):
        # generate test data
        data = os.urandom(data_size)
        # create initial content
        old_content = self.content_factory.new(
            self.container_id, self.content, len(data), self.stgpol)
        # verify factory work as intended
        self.assertEqual(type(old_content), ECContent)

        # perform initial content creation
        old_content.create(StringIO(data))

        uploaded_content = self.content_factory.get(self.container_id,
                                                    old_content.content_id)

        # break the content
        old_info = {}
        for pos in broken_pos_list:
            old_info[pos] = {}
#.........这里部分代码省略.........

# 需要导入模块: from oio.container.client import ContainerClient [as 别名]
# 或者: from oio.container.client.ContainerClient import content_show [as 别名]
class ContentFactory(object):
    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf)
        self.cs_client = ConscienceClient(conf)
        self.container_client = ContainerClient(conf)
        self.ns_info = self.cs_client.info()

    def _extract_datasec(self, stgpol_name):
        try:
            stgpol = self.ns_info["storage_policy"][stgpol_name]
        except KeyError:
            self.logger.error("Storage policy '%s' not found" % stgpol_name)
            raise InconsistentContent("Storage policy not found")

        stgclass_name, datasec_name, datatreat_name = stgpol.split(':')
        if datasec_name == 'NONE':
            return "DUP", {"nb_copy": "1", "distance": "0"}

        try:
            datasec = self.ns_info["data_security"][datasec_name]
        except KeyError:
            self.logger.error("Data security '%s' not found" % datasec_name)
            raise InconsistentContent("Data security not found")

        ds_type, ds_args = datasec.split(':')
        args = {}
        for arg in ds_args.split('|'):
            key, value = arg.split('=')
            args[key] = value

        return ds_type, args

    def get(self, container_id, content_id):
        try:
            meta, chunks = self.container_client.content_show(
                cid=container_id, content=content_id)
        except NotFound:
            raise ContentNotFound("Content %s/%s not found" % (container_id,
                                  content_id))

        pol_type, pol_args = self._extract_datasec(meta['policy'])

        if pol_type == "DUP":
            return DupContent(self.conf, container_id, meta, chunks, pol_args)
        elif pol_type == "RAIN":
            return RainContent(self.conf, container_id, meta, chunks, pol_args)

        raise InconsistentContent("Unknown storage policy")

    def new(self, container_id, path, size, policy):
        meta, chunks = self.container_client.content_prepare(
            cid=container_id, path=path, size=size, stgpol=policy)

        pol_type, pol_args = self._extract_datasec(meta['policy'])

        if pol_type == "DUP":
            return DupContent(self.conf, container_id, meta, chunks, pol_args)
        elif pol_type == "RAIN":
            return RainContent(self.conf, container_id, meta, chunks, pol_args)

        raise InconsistentContent("Unknown storage policy")

    def change_policy(self, container_id, content_id, new_policy):
        old_content = self.get(container_id, content_id)
        if old_content.stgpol_name == new_policy:
            return old_content

        new_content = self.new(container_id, old_content.path,
                               old_content.length, new_policy)

        stream = old_content.download()
        new_content.upload(GeneratorIO(stream))
        # the old content is automatically deleted because the new content has
        # the same name (but not the same id)
        return new_content