Python diskfile.read_metadata函数代码示例

    def run_quarantine_range_etag(self):
        container = 'container-range-%s' % uuid4()
        obj = 'object-range-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj,
                                                        'RANGE')

        metadata = read_metadata(data_file)
        metadata['ETag'] = 'badetag'
        write_metadata(data_file, metadata)
        base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx}
        for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
                               ({'Range': 'bytes=1-11'}, 'ANGE'),
                               ({'Range': 'bytes=0-11'}, 'RANGE')]:
            req_headers = base_headers.copy()
            req_headers.update(header)
            odata = direct_client.direct_get_object(
                onode, opart, self.account, container, obj,
                headers=req_headers)[-1]
            self.assertEqual(odata, result)

        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEqual(err.http_status, 404)

    def run_quarantine(self):
        container = 'container-%s' % uuid4()
        obj = 'object-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj,
                                                        'VERIFY')
        # Stash the on disk data for future comparison - this may not equal
        # 'VERIFY' if for example the proxy has crypto enabled
        backend_data = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        metadata = read_metadata(data_file)
        metadata['ETag'] = 'badetag'
        write_metadata(data_file, metadata)

        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
        self.assertEqual(odata, backend_data)
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEqual(err.http_status, 404)

    def run_quarantine_range_etag(self):
        container = 'container-range-%s' % uuid4()
        obj = 'object-range-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj,
                                                        'RANGE')
        # Stash the on disk data for future comparison - this may not equal
        # 'VERIFY' if for example the proxy has crypto enabled
        backend_data = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={
                'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]

        metadata = read_metadata(data_file)
        metadata['ETag'] = 'badetag'
        write_metadata(data_file, metadata)
        base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx}
        for header, result in [({'Range': 'bytes=0-2'}, backend_data[0:3]),
                               ({'Range': 'bytes=1-11'}, backend_data[1:]),
                               ({'Range': 'bytes=0-11'}, backend_data)]:
            req_headers = base_headers.copy()
            req_headers.update(header)
            odata = direct_client.direct_get_object(
                onode, opart, self.account, container, obj,
                headers=req_headers)[-1]
            self.assertEqual(odata, result)

        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={
                    'X-Backend-Storage-Policy-Index': self.policy.idx})
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEqual(err.http_status, 404)

    def run_quarantine_range_etag(self):
        container = "container-range-%s" % uuid4()
        obj = "object-range-%s" % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, "RANGE")

        metadata = read_metadata(data_file)
        metadata["ETag"] = "badetag"
        write_metadata(data_file, metadata)
        base_headers = {"X-Backend-Storage-Policy-Index": self.policy.idx}
        for header, result in [
            ({"Range": "bytes=0-2"}, "RAN"),
            ({"Range": "bytes=1-11"}, "ANGE"),
            ({"Range": "bytes=0-11"}, "RANGE"),
        ]:
            req_headers = base_headers.copy()
            req_headers.update(header)
            odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=req_headers)[-1]
            self.assertEquals(odata, result)

        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx}
            )
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEquals(err.http_status, 404)

    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = diskfile.DiskFile(self.devices, device, partition,
                                   account, container, obj, self.logger,
                                   keep_data_fp=True)
            try:
                try:
                    obj_size = df.get_data_file_size()
                except DiskFileNotExist:
                    return
                except DiskFileError as e:
                    raise AuditException(str(e))
                if self.stats_sizes:
                    self.record_stats(obj_size)
                if self.zero_byte_only_at_fps and obj_size:
                    self.passes += 1
                    return
                for chunk in df:
                    self.bytes_running_time = ratelimit_sleep(
                        self.bytes_running_time, self.max_bytes_per_second,
                        incr_by=len(chunk))
                    self.bytes_processed += len(chunk)
                    self.total_bytes_processed += len(chunk)
                df.close()
                if df.quarantined_dir:
                    self.quarantines += 1
                    self.logger.error(
                        _("ERROR Object %(path)s failed audit and will be "
                          "quarantined: ETag and file's md5 do not match"),
                        {'path': path})
            finally:
                df.close(verify_file=False)
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and will '
                                'be quarantined: %(err)s'),
                              {'obj': path, 'err': err})
            diskfile.quarantine_renamer(
                os.path.join(self.devices, device), path)
            return
        except (Exception, Timeout):
            self.logger.increment('errors')
            self.errors += 1
            self.logger.exception(_('ERROR Trying to audit %s'), path)
            return
        self.passes += 1

    def object_audit(self, path, device, partition):
        """
        Audits the given object path.

        :param path: a path to an object
        :param device: the device the path is on
        :param partition: the partition the path is on
        """
        try:
            try:
                name = diskfile.read_metadata(path)['name']
            except (Exception, Timeout) as exc:
                raise AuditException('Error when reading metadata: %s' % exc)
            _junk, account, container, obj = name.split('/', 3)
            df = self.diskfile_mgr.get_diskfile(
                device, partition, account, container, obj)
            try:
                with df.open():
                    metadata = df.get_metadata()
                    obj_size = int(metadata['Content-Length'])
                    if self.stats_sizes:
                        self.record_stats(obj_size)
                    if self.zero_byte_only_at_fps and obj_size:
                        self.passes += 1
                        return
                    reader = df.reader()
                with closing(reader):
                    for chunk in reader:
                        chunk_len = len(chunk)
                        self.bytes_running_time = ratelimit_sleep(
                            self.bytes_running_time,
                            self.max_bytes_per_second,
                            incr_by=chunk_len)
                        self.bytes_processed += chunk_len
                        self.total_bytes_processed += chunk_len
                if reader.was_quarantined:
                    self.quarantines += 1
                    self.logger.error(_('ERROR Object %(obj)s failed audit and'
                                        ' was quarantined: %(err)s'),
                                      {'obj': path,
                                       'err': reader.was_quarantined})
                    return
            except DiskFileNotExist:
                return
        except DiskFileQuarantined as err:
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and was'
                                ' quarantined: %(err)s'),
                              {'obj': path, 'err': err})
        except AuditException as err:
            self.logger.increment('quarantines')
            self.quarantines += 1
            self.logger.error(_('ERROR Object %(obj)s failed audit and will'
                                ' be quarantined: %(err)s'),
                              {'obj': path, 'err': err})
            diskfile.quarantine_renamer(
                os.path.join(self.devices, device), path)
            return
        self.passes += 1

    def run_quarantine_zero_byte_head(self):
        container = 'container-zbyte-%s' % uuid4()
        obj = 'object-zbyte-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
        metadata = read_metadata(data_file)
        unlink(data_file)

        with open(data_file, 'w') as fpointer:
            write_metadata(fpointer, metadata)
        try:
            direct_client.direct_head_object(onode, opart, self.account,
                                             container, obj, conn_timeout=1,
                                             response_timeout=1)
            raise Exception("Did not quarantine object")
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine(self):
        container = "container-%s" % uuid4()
        obj = "object-%s" % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY")
        with open(data_file) as fpointer:
            metadata = read_metadata(fpointer)
        metadata["ETag"] = "badetag"
        with open(data_file) as fpointer:
            write_metadata(fpointer, metadata)

        odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1]
        self.assertEquals(odata, "VERIFY")
        try:
            direct_client.direct_get_object(onode, opart, self.account, container, obj)
            raise Exception("Did not quarantine object")
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine_zero_byte_get(self):
        container = 'container-zbyte-%s' % uuid4()
        obj = 'object-zbyte-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
        metadata = read_metadata(data_file)
        unlink(data_file)

        with open(data_file, 'w') as fpointer:
            write_metadata(fpointer, metadata)
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, conn_timeout=1,
                response_timeout=1, headers={'X-Backend-Storage-Policy-Index':
                                             self.policy.idx})
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEqual(err.http_status, 404)

    def run_quarantine(self):
        container = 'container-%s' % uuid4()
        obj = 'object-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj,
                                                        'VERIFY')
        metadata = read_metadata(data_file)
        metadata['ETag'] = 'badetag'
        write_metadata(data_file, metadata)

        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj)[-1]
        self.assertEquals(odata, 'VERIFY')
        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
            raise Exception("Did not quarantine object")
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine(self):
        container = "container-%s" % uuid4()
        obj = "object-%s" % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, "VERIFY")
        metadata = read_metadata(data_file)
        metadata["ETag"] = "badetag"
        write_metadata(data_file, metadata)

        odata = direct_client.direct_get_object(
            onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx}
        )[-1]
        self.assertEquals(odata, "VERIFY")
        try:
            direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers={"X-Backend-Storage-Policy-Index": self.policy.idx}
            )
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine_zero_byte_post(self):
        container = "container-zbyte-%s" % uuid4()
        obj = "object-zbyte-%s" % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, "DATA")
        metadata = read_metadata(data_file)
        unlink(data_file)

        with open(data_file, "w") as fpointer:
            write_metadata(fpointer, metadata)
        try:
            headers = {
                "X-Object-Meta-1": "One",
                "X-Object-Meta-Two": "Two",
                "X-Backend-Storage-Policy-Index": self.policy.idx,
            }
            direct_client.direct_post_object(
                onode, opart, self.account, container, obj, headers=headers, conn_timeout=1, response_timeout=1
            )
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine_zero_byte_post(self):
        container = 'container-zbyte-%s' % uuid4()
        obj = 'object-zbyte-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
        metadata = read_metadata(data_file)
        unlink(data_file)

        with open(data_file, 'w') as fpointer:
            write_metadata(fpointer, metadata)
        try:
            headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two',
                       POLICY_INDEX: self.policy.idx}
            direct_client.direct_post_object(
                onode, opart, self.account,
                container, obj,
                headers=headers,
                conn_timeout=1,
                response_timeout=1)
            raise Exception("Did not quarantine object")
        except ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine_range_etag(self):
        container = "container-range-%s" % uuid4()
        obj = "object-range-%s" % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj, "RANGE")
        with open(data_file) as fpointer:
            metadata = read_metadata(fpointer)
        metadata["ETag"] = "badetag"
        with open(data_file) as fpointer:
            write_metadata(fpointer, metadata)
        for header, result in [
            ({"Range": "bytes=0-2"}, "RAN"),
            ({"Range": "bytes=1-11"}, "ANGE"),
            ({"Range": "bytes=0-11"}, "RANGE"),
        ]:
            odata = direct_client.direct_get_object(onode, opart, self.account, container, obj, headers=header)[-1]
            self.assertEquals(odata, result)

        try:
            direct_client.direct_get_object(onode, opart, self.account, container, obj)
            raise Exception("Did not quarantine object")
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)

    def run_quarantine_range_etag(self):
        container = 'container-range-%s' % uuid4()
        obj = 'object-range-%s' % uuid4()
        onode, opart, data_file = self._setup_data_file(container, obj,
                                                        'RANGE')

        metadata = read_metadata(data_file)
        metadata['ETag'] = 'badetag'
        write_metadata(data_file, metadata)
        for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
                               ({'Range': 'bytes=1-11'}, 'ANGE'),
                               ({'Range': 'bytes=0-11'}, 'RANGE')]:
            odata = direct_client.direct_get_object(
                onode, opart, self.account, container, obj, headers=header)[-1]
            self.assertEquals(odata, result)

        try:
            direct_client.direct_get_object(onode, opart, self.account,
                                            container, obj)
            raise Exception("Did not quarantine object")
        except client.ClientException as err:
            self.assertEquals(err.http_status, 404)