Python constraints.check_utf8函数代码示例

    def validate_metadata(metadata):
        """
        Validates that metadata falls within acceptable limits.

        :param metadata: to be validated
        :raises HTTPBadRequest: if MAX_META_COUNT or MAX_META_OVERALL_SIZE
                 is exceeded, or if metadata contains non-UTF-8 data
        """
        meta_count = 0
        meta_size = 0
        for key, (value, timestamp) in metadata.items():
            if key and not isinstance(key, six.text_type):
                if not check_utf8(key):
                    raise HTTPBadRequest('Metadata must be valid UTF-8')
                # Promote to a natural string for the checks below
                if six.PY3:
                    key = key.decode('utf8')
            if value and not isinstance(value, six.text_type):
                if not check_utf8(value):
                    raise HTTPBadRequest('Metadata must be valid UTF-8')
            key = key.lower()
            if value and key.startswith(('x-account-meta-',
                                         'x-container-meta-')):
                prefix = 'x-account-meta-'
                if key.startswith('x-container-meta-'):
                    prefix = 'x-container-meta-'
                key = key[len(prefix):]
                meta_count = meta_count + 1
                meta_size = meta_size + len(key) + len(value)
        if meta_count > MAX_META_COUNT:
            raise HTTPBadRequest('Too many metadata items; max %d'
                                 % MAX_META_COUNT)
        if meta_size > MAX_META_OVERALL_SIZE:
            raise HTTPBadRequest('Total metadata too large; max %d'
                                 % MAX_META_OVERALL_SIZE)

    def validate_metadata(metadata):
        """
        Validates that metadata falls within acceptable limits.

        :param metadata: to be validated
        :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE
                 is exceeded, or if metadata contains non-UTF-8 data
        """
        meta_count = 0
        meta_size = 0
        for key, (value, timestamp) in metadata.items():
            key = key.lower()
            if value != '' and (key.startswith('x-account-meta') or
                                key.startswith('x-container-meta')):
                prefix = 'x-account-meta-'
                if key.startswith('x-container-meta-'):
                    prefix = 'x-container-meta-'
                key = key[len(prefix):]
                meta_count = meta_count + 1
                meta_size = meta_size + len(key) + len(value)
            bad_key = key and not check_utf8(key)
            bad_value = value and not check_utf8(value)
            if bad_key or bad_value:
                raise HTTPBadRequest('Metadata must be valid UTF-8')
        if meta_count > MAX_META_COUNT:
            raise HTTPBadRequest('Too many metadata items; max %d'
                                 % MAX_META_COUNT)
        if meta_size > MAX_META_OVERALL_SIZE:
            raise HTTPBadRequest('Total metadata too large; max %d'
                                 % MAX_META_OVERALL_SIZE)

 def PUT(self, req):
     """Handle HTTP PUT request."""
     try:
         drive, part, account, container = split_path(unquote(req.path),
                                                      3, 4)
         if (account and not check_utf8(account)) or \
             (container and not check_utf8(container)):
             raise ValueError('NULL characters not allowed in names')
     except ValueError, err:
         return HTTPBadRequest(body=str(err), content_type='text/plain',
                               request=req)

    def test_check_utf8(self):
        unicode_sample = u"\uc77c\uc601"
        valid_utf8_str = unicode_sample.encode("utf-8")
        invalid_utf8_str = unicode_sample.encode("utf-8")[::-1]
        unicode_with_null = u"abc\u0000def"
        utf8_with_null = unicode_with_null.encode("utf-8")

        for false_argument in [None, "", invalid_utf8_str, unicode_with_null, utf8_with_null]:
            self.assertFalse(constraints.check_utf8(false_argument))

        for true_argument in ["this is ascii and utf-8, too", unicode_sample, valid_utf8_str]:
            self.assertTrue(constraints.check_utf8(true_argument))

    def test_check_utf8(self):
        unicode_sample = u'\uc77c\uc601'
        valid_utf8_str = unicode_sample.encode('utf-8')
        invalid_utf8_str = unicode_sample.encode('utf-8')[::-1]

        for false_argument in [None,
                               '',
                               invalid_utf8_str,
                               ]:
            self.assertFalse(constraints.check_utf8(false_argument))

        for true_argument in ['this is ascii and utf-8, too',
                              unicode_sample,
                              valid_utf8_str]:
            self.assertTrue(constraints.check_utf8(true_argument))

 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get("x-trans-id", None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL")
     else:
         try:
             # disallow methods which are not publicly accessible
             try:
                 if req.method not in self.allowed_methods:
                     raise AttributeError("Not allowed method.")
             except AttributeError:
                 res = HTTPMethodNotAllowed()
             else:
                 method = getattr(self, req.method)
                 res = method(req)
         except HTTPException as error_response:
             res = error_response
         except (Exception, Timeout):
             self.logger.exception(
                 _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path}
             )
             res = HTTPInternalServerError(body=traceback.format_exc())
     if self.log_requests:
         trans_time = time.time() - start_time
         additional_info = ""
         if res.headers.get("x-container-timestamp") is not None:
             additional_info += "x-container-timestamp: %s" % res.headers["x-container-timestamp"]
         log_msg = get_log_line(req, res, trans_time, additional_info)
         if req.method.upper() == "REPLICATE":
             self.logger.debug(log_msg)
         else:
             self.logger.info(log_msg)
     return res(env, start_response)

    def handle_request(self, req):

        try:
            self.logger.set_statsd_prefix('proxy-server')
            if req.content_length and req.content_length < 0:
                return jresponse('-1','Invalid Content-Length',req,400)
                
            try:
                if not check_utf8(req.path_info):
                    
                    return jresponse('-1','Invalid UTF8',req,412)
            except UnicodeError:
                return jresponse('-1','Invalid UTF8',req,412)
                
            
            try:
                controller, path_parts = self.get_controller(req)
                p = req.path_info
                if isinstance(p, unicode):
                    p = p.encode('utf-8')
            except ValueError:
                return jresponse('-1','not found',req,404)
            if not controller:
                return jresponse('-1','Bad URL',req,412)
            
            if self.deny_host_headers and \
                    req.host.split(':')[0] in self.deny_host_headers:
                return HTTPForbidden(request=req, body='Invalid host header')
            if not check_path_parts(path_parts):
                return HTTPForbidden(request=req, body='Invalid path_parts header')
            
            self.logger.set_statsd_prefix('proxy-server.' +
                                          controller.server_type.lower())
            
            controller = controller(self, **path_parts)
            if 'swift.trans_id' not in req.environ:
                # if this wasn't set by an earlier middleware, set it now
                trans_id = 'tx' + uuid.uuid4().hex
                req.environ['swift.trans_id'] = trans_id
                self.logger.txn_id = trans_id
            req.headers['x-trans-id'] = req.environ['swift.trans_id']
            controller.trans_id = req.environ['swift.trans_id']
            self.logger.client_ip = get_remote_client(req)
            
            try:
                if req.GET.get('op'):
                    req.method = req.GET.get('op')
                    
                handler = getattr(controller, req.method)
                getattr(handler, 'publicly_accessible')
            except AttributeError:
                return HTTPMethodNotAllowed(request=req)
            if path_parts['version']:
                req.path_info_pop()
    
            req.environ['swift.orig_req_method'] = req.method
            return handler(req)
        except (Exception, Timeout):
            self.logger.exception(_('ERROR Unhandled exception in request'))
            return jresponse('-1','ServerERROR',req,500)

 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8')
     else:
         try:
             if hasattr(self, req.method):
                 res = getattr(self, req.method)(req)
             else:
                 res = HTTPMethodNotAllowed()
         except (Exception, Timeout):
             self.logger.exception(_('ERROR __call__ error with %(method)s'
                 ' %(path)s '), {'method': req.method, 'path': req.path})
             res = HTTPInternalServerError(body=traceback.format_exc())
     trans_time = '%.4f' % (time.time() - start_time)
     additional_info = ''
     if res.headers.get('x-container-timestamp') is not None:
         additional_info += 'x-container-timestamp: %s' % \
             res.headers['x-container-timestamp']
     log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % (
         req.remote_addr,
         time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()),
         req.method, req.path,
         res.status.split()[0], res.content_length or '-',
         req.headers.get('x-trans-id', '-'),
         req.referer or '-', req.user_agent or '-',
         trans_time,
         additional_info)
     if req.method.upper() == 'REPLICATE':
         self.logger.debug(log_message)
     else:
         self.logger.info(log_message)
     return res(env, start_response)

 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8')
     else:
         try:
             # disallow methods which have not been marked 'public'
             try:
                 method = getattr(self, req.method)
                 getattr(method, 'publicly_accessible')
             except AttributeError:
                 res = HTTPMethodNotAllowed()
             else:
                 res = method(req)
         except (Exception, Timeout):
             self.logger.exception(_('ERROR __call__ error with %(method)s'
                 ' %(path)s '), {'method': req.method, 'path': req.path})
             res = HTTPInternalServerError(body=traceback.format_exc())
     trans_time = '%.4f' % (time.time() - start_time)
     log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
         req.remote_addr,
         time.strftime('%d/%b/%Y:%H:%M:%S +0000',
                       time.gmtime()),
         req.method, req.path,
         res.status.split()[0], res.content_length or '-',
         req.headers.get('x-trans-id', '-'),
         req.referer or '-', req.user_agent or '-',
         trans_time)
     if req.method.upper() == 'REPLICATE':
         self.logger.debug(log_message)
     else:
         self.logger.info(log_message)
     return res(env, start_response)

    def __call__(self, env, start_response):

        req = Request(env)

        if 'd' != req.headers.get('X-Ftype'):
            return self.app(env,start_response)

        self.logger.txn_id = req.headers.get('x-trans-id', None)
        
        if not check_utf8(req.path_info):
            res = jresponse('-1', 'invalid UTF8', req,412)
            
        else:
            try:
                # disallow methods which have not been marked 'public'
                try:
                    method = getattr(self, req.method)
                    getattr(method, 'publicly_accessible')
                except AttributeError:
                    res = jresponse('-1', 'method not allowed', req,405)
                else:
                    res = method(req)

            except (Exception, Timeout):
                self.logger.exception(_('ERROR __call__ error with %(method)s'
                    ' %(path)s '), {'method': req.method, 'path': req.path})
                res = jresponse('-1', 'internal server error', req,500)
                
        return res(env, start_response)

 def __call__(self, env, start_response):
     
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = jresponse('-1','Invalid UTF8',req,412)
     else:
         try:
             # disallow methods which have not been marked 'public'
             try:
                 method = getattr(self, req.method)
                 getattr(method, 'publicly_accessible')
             except AttributeError:
                 res = jresponse('-1', 'method not allowed', req,405) 
             else:
                 res = method(req)
                 # if req.method == 'PUT':
                 #    print 'path:   '+req.path +  '      status:  '+str(res.status_int) + '  msg: '+res.body
         except (Exception, Timeout):
             self.logger.exception(_('ERROR __call__ error with %(method)s'
                 ' %(path)s '), {'method': req.method, 'path': req.path})
             res = jresponse('-1', 'InternalServerError', req,500)
     
     return res(env, start_response)

    def get_segments_to_delete_iter(self, req):
        """
        A generator function to be used to delete all the segments and
        sub-segments referenced in a manifest.

        :params req: a swob.Request with an SLO manifest in path
        :raises HTTPPreconditionFailed: on invalid UTF8 in request path
        :raises HTTPBadRequest: on too many buffered sub segments and
                                on invalid SLO manifest path
        """
        if not check_utf8(req.path_info):
            raise HTTPPreconditionFailed(request=req, body="Invalid UTF8 or contains NULL")
        vrs, account, container, obj = req.split_path(4, 4, True)

        segments = [{"sub_slo": True, "name": ("/%s/%s" % (container, obj)).decode("utf-8")}]
        while segments:
            if len(segments) > MAX_BUFFERED_SLO_SEGMENTS:
                raise HTTPBadRequest("Too many buffered slo segments to delete.")
            seg_data = segments.pop(0)
            if seg_data.get("sub_slo"):
                try:
                    segments.extend(self.get_slo_segments(seg_data["name"], req))
                except HTTPException as err:
                    # allow bulk delete response to report errors
                    seg_data["error"] = {"code": err.status_int, "message": err.body}

                # add manifest back to be deleted after segments
                seg_data["sub_slo"] = False
                segments.append(seg_data)
            else:
                seg_data["name"] = seg_data["name"].encode("utf-8")
                yield seg_data

 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get('x-trans-id', None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
     else:
         try:
             # disallow methods which are not publicly accessible
             if req.method not in self.allowed_methods:
                 res = HTTPMethodNotAllowed()
             else:
                 res = getattr(self, req.method)(req)
         except HTTPException as error_response:
             res = error_response
         except (Exception, Timeout):
             self.logger.exception(_('ERROR __call__ error with %(method)s'
                                     ' %(path)s '),
                                   {'method': req.method, 'path': req.path})
             res = HTTPInternalServerError(body=traceback.format_exc())
     if self.log_requests:
         trans_time = time.time() - start_time
         additional_info = ''
         if res.headers.get('x-container-timestamp') is not None:
             additional_info += 'x-container-timestamp: %s' % \
                 res.headers['x-container-timestamp']
         log_msg = get_log_line(req, res, trans_time, additional_info,
                                self.log_format, self.anonymization_method,
                                self.anonymization_salt)
         if req.method.upper() == 'REPLICATE':
             self.logger.debug(log_msg)
         else:
             self.logger.info(log_msg)
     return res(env, start_response)

 def __call__(self, env, start_response):
     start_time = time.time()
     req = Request(env)
     self.logger.txn_id = req.headers.get("x-trans-id", None)
     if not check_utf8(req.path_info):
         res = HTTPPreconditionFailed(body="Invalid UTF8")
     else:
         try:
             if hasattr(self, req.method):
                 res = getattr(self, req.method)(req)
             else:
                 res = HTTPMethodNotAllowed()
         except (Exception, Timeout):
             self.logger.exception(
                 _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path}
             )
             res = HTTPInternalServerError(body=traceback.format_exc())
     trans_time = "%.4f" % (time.time() - start_time)
     log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
         req.remote_addr,
         time.strftime("%d/%b/%Y:%H:%M:%S +0000", time.gmtime()),
         req.method,
         req.path,
         res.status.split()[0],
         res.content_length or "-",
         req.headers.get("x-trans-id", "-"),
         req.referer or "-",
         req.user_agent or "-",
         trans_time,
     )
     if req.method.upper() == "REPLICATE":
         self.logger.debug(log_message)
     else:
         self.logger.info(log_message)
     return res(env, start_response)

    def handle_extract(self, req, compress_type):
        """
        :params req: a swob Request
        :params compress_type: specifying the compression type of the tar.
                               Accepts '', 'gz, or 'bz2'
        :raises HTTPException: on unhandled errors
        :returns: a swob response to request
        """
        success_count = 0
        failed_files = []
        existing_containers = set()
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            return HTTPNotAcceptable(request=req)
        if req.content_length is None and req.headers.get("transfer-encoding", "").lower() != "chunked":
            return HTTPBadRequest("Invalid request: no content sent.")
        try:
            vrs, account, extract_base = req.split_path(2, 3, True)
        except ValueError:
            return HTTPNotFound(request=req)
        extract_base = extract_base or ""
        extract_base = extract_base.rstrip("/")
        try:
            tar = tarfile.open(mode="r|" + compress_type, fileobj=req.body_file)
            while True:
                tar_info = tar.next()
                if tar_info is None or len(failed_files) >= self.max_failed_extractions:
                    break
                if tar_info.isfile():
                    obj_path = tar_info.name
                    if obj_path.startswith("./"):
                        obj_path = obj_path[2:]
                    obj_path = obj_path.lstrip("/")
                    if extract_base:
                        obj_path = extract_base + "/" + obj_path
                    if "/" not in obj_path:
                        continue  # ignore base level file

                    destination = "/".join(["", vrs, account, obj_path])
                    container = obj_path.split("/", 1)[0]
                    if not check_utf8(destination):
                        failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTPPreconditionFailed().status])
                        continue
                    if tar_info.size > MAX_FILE_SIZE:
                        failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTPRequestEntityTooLarge().status])
                        continue
                    if container not in existing_containers:
                        try:
                            self.create_container(req, "/".join(["", vrs, account, container]))
                            existing_containers.add(container)
                        except CreateContainerError, err:
                            if err.status_int == HTTP_UNAUTHORIZED:
                                return HTTPUnauthorized(request=req)
                            failed_files.append([quote(destination[:MAX_PATH_LENGTH]), err.status])
                            continue
                        except ValueError:
                            failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTP_BAD_REQUEST])
                            continue
                        if len(existing_containers) > self.max_containers:
                            return HTTPBadRequest("More than %d base level containers in tar." % self.max_containers)