Python Bulk.handle_delete_iter方法代码示例

# 需要导入模块: from swift.common.middleware.bulk import Bulk [as 别名]
# 或者: from swift.common.middleware.bulk.Bulk import handle_delete_iter [as 别名]

#.........这里部分代码省略.........

                # add manifest back to be deleted after segments
                seg_data['sub_slo'] = False
                segments.append(seg_data)
            else:
                seg_data['name'] = seg_data['name'].encode('utf-8')
                yield seg_data

    def get_slo_segments(self, obj_name, req):
        """
        Performs a swob.Request and returns the SLO manifest's segments.

        :raises HTTPServerError: on unable to load obj_name or
                                 on unable to load the SLO manifest data.
        :raises HTTPBadRequest: on not an SLO manifest
        :raises HTTPNotFound: on SLO manifest not found
        :returns: SLO manifest's segments
        """
        vrs, account, _junk = req.split_path(2, 3, True)
        new_env = req.environ.copy()
        new_env['REQUEST_METHOD'] = 'GET'
        del(new_env['wsgi.input'])
        new_env['QUERY_STRING'] = 'multipart-manifest=get'
        new_env['CONTENT_LENGTH'] = 0
        new_env['HTTP_USER_AGENT'] = \
            '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')
        new_env['swift.source'] = 'SLO'
        new_env['PATH_INFO'] = (
            '/%s/%s/%s' % (vrs, account, obj_name.lstrip('/'))
        ).encode('utf-8')
        resp = Request.blank('', new_env).get_response(self.app)

        if resp.is_success:
            if config_true_value(resp.headers.get('X-Static-Large-Object')):
                try:
                    return json.loads(resp.body)
                except ValueError:
                    raise HTTPServerError('Unable to load SLO manifest')
            else:
                raise HTTPBadRequest('Not an SLO manifest')
        elif resp.status_int == HTTP_NOT_FOUND:
            raise HTTPNotFound('SLO manifest not found')
        elif resp.status_int == HTTP_UNAUTHORIZED:
            raise HTTPUnauthorized('401 Unauthorized')
        else:
            raise HTTPServerError('Unable to load SLO manifest or segment.')

    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.

        :params req: a swob.Request with an obj in path
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        req.headers['Content-Type'] = None  # Ignore content-type from client
        resp = HTTPOk(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if out_content_type:
            resp.content_type = out_content_type
        resp.app_iter = self.bulk_deleter.handle_delete_iter(
            req, objs_to_delete=self.get_segments_to_delete_iter(req),
            user_agent='MultipartDELETE', swift_source='SLO',
            out_content_type=out_content_type)
        return resp

    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        try:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req, start_response)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == 'GET' or req.method == 'HEAD':
                return self.handle_multipart_get_or_head(req, start_response)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)

# 需要导入模块: from swift.common.middleware.bulk import Bulk [as 别名]
# 或者: from swift.common.middleware.bulk.Bulk import handle_delete_iter [as 别名]

#.........这里部分代码省略.........
                    segments.extend(
                        self.get_slo_segments(seg_data['name'], req))
                except HTTPException as err:
                    # allow bulk delete response to report errors
                    seg_data['error'] = {'code': err.status_int,
                                         'message': err.body}

                # add manifest back to be deleted after segments
                seg_data['sub_slo'] = False
                segments.append(seg_data)
            else:
                seg_data['name'] = seg_data['name'].encode('utf-8')
                yield seg_data

    def get_slo_segments(self, obj_name, req):
        """
        Performs a swob.Request and returns the SLO manifest's segments.

        :raises HTTPServerError: on unable to load obj_name or
                                 on unable to load the SLO manifest data.
        :raises HTTPBadRequest: on not an SLO manifest
        :raises HTTPNotFound: on SLO manifest not found
        :returns: SLO manifest's segments
        """
        vrs, account, _junk = req.split_path(2, 3, True)
        new_env = req.environ.copy()
        new_env['REQUEST_METHOD'] = 'GET'
        del(new_env['wsgi.input'])
        new_env['QUERY_STRING'] = 'multipart-manifest=get'
        new_env['CONTENT_LENGTH'] = 0
        new_env['HTTP_USER_AGENT'] = \
            '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT')
        new_env['swift.source'] = 'SLO'
        new_env['PATH_INFO'] = (
            '/%s/%s/%s' % (
            vrs, account,
            obj_name.lstrip('/'))).encode('utf-8')
        resp = Request.blank('', new_env).get_response(self.app)

        if resp.is_success:
            if config_true_value(resp.headers.get('X-Static-Large-Object')):
                try:
                    return json.loads(resp.body)
                except ValueError:
                    raise HTTPServerError('Unable to load SLO manifest')
            else:
                raise HTTPBadRequest('Not an SLO manifest')
        elif resp.status_int == HTTP_NOT_FOUND:
            raise HTTPNotFound('SLO manifest not found')
        elif resp.status_int == HTTP_UNAUTHORIZED:
            raise HTTPUnauthorized('401 Unauthorized')
        else:
            raise HTTPServerError('Unable to load SLO manifest or segment.')

    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.

        :params req: a swob.Request with an obj in path
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        resp = HTTPOk(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if out_content_type:
            resp.content_type = out_content_type
        resp.app_iter = self.bulk_deleter.handle_delete_iter(
            req, objs_to_delete=self.get_segments_to_delete_iter(req),
            user_agent='MultipartDELETE', swift_source='SLO',
            out_content_type=out_content_type)
        return resp

    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        try:
            if obj:
                if req.method == 'PUT' and \
                        req.params.get('multipart-manifest') == 'put':
                    return self.handle_multipart_put(req, start_response)
                if req.method == 'DELETE' and \
                        req.params.get('multipart-manifest') == 'delete':
                    return self.handle_multipart_delete(req)(env,
                                                             start_response)
                if 'X-Static-Large-Object' in req.headers:
                    raise HTTPBadRequest(
                        request=req,
                        body='X-Static-Large-Object is a reserved header. '
                        'To create a static large object add query param '
                        'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)

# 需要导入模块: from swift.common.middleware.bulk import Bulk [as 别名]
# 或者: from swift.common.middleware.bulk.Bulk import handle_delete_iter [as 别名]

#.........这里部分代码省略.........
                    # shouldn't happen
                    last_modified = datetime.now()

                last_modified_formatted = \
                    last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')
                data_for_storage.append(
                    {'name': '/' + seg_dict['path'].lstrip('/'),
                     'bytes': seg_size,
                     'hash': seg_dict['etag'],
                     'content_type': head_seg_resp.content_type,
                     'last_modified': last_modified_formatted})

            else:
                problem_segments.append([quote(obj_path),
                                         head_seg_resp.status])
        if problem_segments:
            resp_body = get_response_body(
                out_content_type, {}, problem_segments)
            raise HTTPBadRequest(resp_body, content_type=out_content_type)
        env = req.environ

        if not env.get('CONTENT_TYPE'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream'
        env['swift.content_type_overriden'] = True
        env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size
        env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True'
        json_data = json.dumps(data_for_storage)
        env['CONTENT_LENGTH'] = str(len(json_data))
        env['wsgi.input'] = StringIO(json_data)
        return self.app

    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.
        :params req: a swob.Request with an obj in path
        :raises HTTPServerError: on invalid manifest
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        if not check_utf8(req.path_info):
            raise HTTPPreconditionFailed(
                request=req, body='Invalid UTF8 or contains NULL')
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            raise HTTPBadRequest('Not an SLO manifest')
        new_env = req.environ.copy()
        new_env['REQUEST_METHOD'] = 'GET'
        del(new_env['wsgi.input'])
        new_env['QUERY_STRING'] = 'multipart-manifest=get'
        new_env['CONTENT_LENGTH'] = 0
        new_env['HTTP_USER_AGENT'] = \
            '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT')
        new_env['swift.source'] = 'SLO'
        get_man_resp = \
            Request.blank('', new_env).get_response(self.app)
        if get_man_resp.status_int // 100 == 2:
            if not config_true_value(
                    get_man_resp.headers.get('X-Static-Large-Object')):
                raise HTTPBadRequest('Not an SLO manifest')
            try:
                manifest = json.loads(get_man_resp.body)
                # append the manifest file for deletion at the end
                manifest.append(
                    {'name': '/'.join(['', container, obj]).decode('utf-8')})
            except ValueError:
                raise HTTPServerError('Invalid manifest file')
            resp = HTTPOk(request=req)
            resp.app_iter = self.bulk_deleter.handle_delete_iter(
                req,
                objs_to_delete=[o['name'].encode('utf-8') for o in manifest],
                user_agent='MultipartDELETE', swift_source='SLO')
            return resp
        return get_man_resp

    @wsgify
    def __call__(self, req):
        """
        WSGI entry point
        """
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app
        if obj:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')

        return self.app

# 需要导入模块: from swift.common.middleware.bulk import Bulk [as 别名]
# 或者: from swift.common.middleware.bulk.Bulk import handle_delete_iter [as 别名]

#.........这里部分代码省略.........
                    # allow bulk delete response to report errors
                    seg_data["error"] = {"code": err.status_int, "message": err.body}

                # add manifest back to be deleted after segments
                seg_data["sub_slo"] = False
                segments.append(seg_data)
            else:
                seg_data["name"] = seg_data["name"].encode("utf-8")
                yield seg_data

    def get_slo_segments(self, obj_name, req):
        """
        Performs a swob.Request and returns the SLO manifest's segments.

        :raises HTTPServerError: on unable to load obj_name or
                                 on unable to load the SLO manifest data.
        :raises HTTPBadRequest: on not an SLO manifest
        :raises HTTPNotFound: on SLO manifest not found
        :returns: SLO manifest's segments
        """
        vrs, account, _junk = req.split_path(2, 3, True)
        new_env = req.environ.copy()
        new_env["REQUEST_METHOD"] = "GET"
        del (new_env["wsgi.input"])
        new_env["QUERY_STRING"] = "multipart-manifest=get"
        new_env["CONTENT_LENGTH"] = 0
        new_env["HTTP_USER_AGENT"] = "%s MultipartDELETE" % new_env.get("HTTP_USER_AGENT")
        new_env["swift.source"] = "SLO"
        new_env["PATH_INFO"] = ("/%s/%s/%s" % (vrs, account, obj_name.lstrip("/"))).encode("utf-8")
        resp = Request.blank("", new_env).get_response(self.app)

        if resp.is_success:
            if config_true_value(resp.headers.get("X-Static-Large-Object")):
                try:
                    return json.loads(resp.body)
                except ValueError:
                    raise HTTPServerError("Unable to load SLO manifest")
            else:
                raise HTTPBadRequest("Not an SLO manifest")
        elif resp.status_int == HTTP_NOT_FOUND:
            raise HTTPNotFound("SLO manifest not found")
        elif resp.status_int == HTTP_UNAUTHORIZED:
            raise HTTPUnauthorized("401 Unauthorized")
        else:
            raise HTTPServerError("Unable to load SLO manifest or segment.")

    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.

        :params req: a swob.Request with an obj in path
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        resp = HTTPOk(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if out_content_type:
            resp.content_type = out_content_type
        resp.app_iter = self.bulk_deleter.handle_delete_iter(
            req,
            objs_to_delete=self.get_segments_to_delete_iter(req),
            user_agent="MultipartDELETE",
            swift_source="SLO",
            out_content_type=out_content_type,
        )
        return resp

    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env["swift.copy_hook"] = self.copy_hook(
            env.get("swift.copy_hook", lambda src_req, src_resp, sink_req: src_resp)
        )

        try:
            if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                return self.handle_multipart_put(req, start_response)
            if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == "GET" or req.method == "HEAD":
                return self.handle_multipart_get_or_head(req, start_response)
            if "X-Static-Large-Object" in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body="X-Static-Large-Object is a reserved header. "
                    "To create a static large object add query param "
                    "multipart-manifest=put.",
                )
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)

# 需要导入模块: from swift.common.middleware.bulk import Bulk [as 别名]
# 或者: from swift.common.middleware.bulk.Bulk import handle_delete_iter [as 别名]

#.........这里部分代码省略.........
        env["CONTENT_TYPE"] += ";swift_bytes=%d" % total_size
        env["HTTP_X_STATIC_LARGE_OBJECT"] = "True"
        json_data = json.dumps(data_for_storage)
        env["CONTENT_LENGTH"] = str(len(json_data))
        env["wsgi.input"] = StringIO(json_data)
        return self.app

    def get_segments_to_delete_iter(self, req):
        """
        A generator function to be used to delete all the segments and
        sub-segments referenced in a manifest.

        :raises HTTPBadRequest: on sub manifest not manifest anymore or
                                on too many buffered sub segments
        :raises HTTPServerError: on unable to load manifest
        """
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            raise HTTPBadRequest("Not a SLO manifest")
        sub_segments = [{"sub_slo": True, "name": ("/%s/%s" % (container, obj)).decode("utf-8")}]
        while sub_segments:
            if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS:
                raise HTTPBadRequest("Too many buffered slo segments to delete.")
            if sub_segments:
                seg_data = sub_segments.pop(0)
            if seg_data.get("sub_slo"):
                new_env = req.environ.copy()
                new_env["REQUEST_METHOD"] = "GET"
                del (new_env["wsgi.input"])
                new_env["QUERY_STRING"] = "multipart-manifest=get"
                new_env["CONTENT_LENGTH"] = 0
                new_env["HTTP_USER_AGENT"] = "%s MultipartDELETE" % new_env.get("HTTP_USER_AGENT")
                new_env["swift.source"] = "SLO"
                new_env["PATH_INFO"] = ("/%s/%s/%s" % (vrs, account, seg_data["name"].lstrip("/"))).encode("utf-8")
                sub_resp = Request.blank("", new_env).get_response(self.app)
                if sub_resp.is_success:
                    try:
                        # if its still a SLO, load its segments
                        if config_true_value(sub_resp.headers.get("X-Static-Large-Object")):
                            sub_segments.extend(json.loads(sub_resp.body))
                    except ValueError:
                        raise HTTPServerError("Unable to load SLO manifest")
                    # add sub-manifest back to be deleted after sub segments
                    # (even if obj is not a SLO)
                    seg_data["sub_slo"] = False
                    sub_segments.append(seg_data)
                elif sub_resp.status_int != HTTP_NOT_FOUND:
                    # on deletes treat not found as success
                    raise HTTPServerError("Sub SLO unable to load.")
            else:
                yield seg_data["name"].encode("utf-8")

    def handle_multipart_delete(self, req):
        """
        Will delete all the segments in the SLO manifest and then, if
        successful, will delete the manifest file.
        :params req: a swob.Request with an obj in path
        :raises HTTPServerError: on invalid manifest
        :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
        """
        if not check_utf8(req.path_info):
            raise HTTPPreconditionFailed(request=req, body="Invalid UTF8 or contains NULL")

        resp = HTTPOk(request=req)
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if out_content_type:
            resp.content_type = out_content_type
        resp.app_iter = self.bulk_deleter.handle_delete_iter(
            req,
            objs_to_delete=self.get_segments_to_delete_iter(req),
            user_agent="MultipartDELETE",
            swift_source="SLO",
            out_content_type=out_content_type,
        )
        return resp

    @wsgify
    def __call__(self, req):
        """
        WSGI entry point
        """
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app
        if obj:
            if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                return self.handle_multipart_put(req)
            if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                return self.handle_multipart_delete(req)
            if "X-Static-Large-Object" in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body="X-Static-Large-Object is a reserved header. "
                    "To create a static large object add query param "
                    "multipart-manifest=put.",
                )

        return self.app