Python utils.get_encoding_from_headers函数代码示例

def find_encoding(content, headers=None):
    # content is unicode
    if isinstance(content, unicode):
        return 'unicode'

    encoding = None

    # Try charset from content-type
    if headers:
        encoding = get_encoding_from_headers(headers)
        if encoding == 'ISO-8859-1':
            encoding = None

    # Try charset from content
    if not encoding:
        encoding = get_encodings_from_content(content)
        encoding = encoding and encoding[0] or None

    # Fallback to auto-detected encoding.
    if not encoding and chardet is not None:
        encoding = chardet.detect(content)['encoding']

    if encoding and encoding.lower() == 'gb2312':
        encoding = 'gb18030'

    return encoding or 'latin_1'

    def _receive_response(self, task, response):
        """
        Called by the delegate when a response has been received.

        This call is expected only on background threads, and thus may not do
        anything that is not Python-thread-safe. This means that, for example,
        it is safe to grab things from the _tasks dictionary, but it is not
        safe to make other method calls on this object unless they explicitly
        state that they are safe in background threads.
        """
        queue, request = self._tasks[task]

        resp = Response()
        resp.status_code = getKey(response, 'statusCode')
        resp.reason = ''

        # TODO: Why do I have to do this?
        raw_headers = getKey(response, 'allHeaderFields')
        resp.headers = CaseInsensitiveDict(raw_headers)
        resp.encoding = get_encoding_from_headers(resp.headers)

        # TODO: This needs to point to an object that we can use to provide
        # the various raw things that requests needs.
        resp.raw = None

        if isinstance(request.url, bytes):
            resp.url = request.url.decode('utf-8')
        else:
            resp.url = request.url

        resp.request = request
        resp.connection = self

        # Put this response on the queue.
        queue.put_nowait(resp)

    def encoding(self):
        if hasattr(self, '_encoding'):
            return self._encoding

        # content is unicode
        if isinstance(self.content, unicode):
            return 'unicode'

        # Try charset from content-type
        encoding = get_encoding_from_headers(self.headers)
        if encoding == 'ISO-8859-1':
            encoding = None

        # Try charset from content
        if not encoding:
            encoding = get_encodings_from_content(self.content)
            encoding = encoding and encoding[0] or None

        # Fallback to auto-detected encoding.
        if not encoding and chardet is not None:
            encoding = chardet.detect(self.content)['encoding']

        if encoding and encoding.lower() == 'gb2312':
            encoding = 'gb18030'

        self._encoding = encoding or 'utf-8'
        return self._encoding

def get_unicode_from_request(r):
    """Returns the requested content back in unicode.

    :param r: Request object to get unicode content from.

    Try:
    1. charset from content-type
    2. fall back and assume utf-8
    3. latin-1 replacing unicode chars 
    """
    if isinstance(r.body, unicode):
        return r.body

    tried_encodings = []

    # Try charset from content-type
    # i.e. "Content-type: text/plain; charset=us-ascii"
    encoding = get_encoding_from_headers(CaseInsensitiveDict(r.headers))

    if encoding:
        try:
            return unicode(r.body, encoding)
        except UnicodeError:
            tried_encodings.append(encoding)

    # workaround if encoding is not specified, assume utf-8, then latin-1                 
    try:
        return unicode(r.body, 'utf-8')
    except:
        tried_encodings.append('utf-8')
        return unicode(r.body, 'latin-1', errors='replace')

def build_response(request,
                   status_code=200,
                   headers={},
                   content='(none)'):
    """
    Build a :class:`requests.Response` object on the basis of the passed
    parameters.
    """

    response = Response()

    response.status_code = status_code
    response.reason = responses[status_code]
    response.headers = CaseInsensitiveDict(headers)
    # Pretend that we've already read from the socket
    response._content = content

    response.encoding = get_encoding_from_headers(response.headers)
    response.url = request.url
    response.raw = MockRawResponse()

    # Give the Response some context.
    response.request = request
    response.connection = MockConnection()

    return response

    def build_response(self, req, resp):
        """Builds a :class:`Response <requests.Response>` object from a urllib3
        response. This should not be called from user code, and is only exposed
        for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`

        :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
        :param resp: The urllib3 response object.
        """
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(req.url, bytes):
            response.url = req.url.decode('utf-8')
        else:
            response.url = req.url

        # Add new cookies from the server.
        extract_cookies_to_jar(response.cookies, req, resp)

        # Give the Response some context.
        response.request = req
        response.connection = self

        return response

    def send(self, request, **kwargs):
        url = urlparse(request.url)
        if url.scheme != 'https':
            raise Exception('Only HTTPS is supported!')

        ctx = self._make_context()

        conn = httpslib.HTTPSConnection(
                url.hostname, url.port or 443, ssl_context=ctx)
        conn.request(request.method, url.path, request.body, request.headers)

        resp = conn.getresponse()
        response = Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)

        # Make headers case-insensitive.
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))

        # Set encoding.
        response.encoding = get_encoding_from_headers(response.headers)
        response.raw = resp
        response.reason = response.raw.reason

        if isinstance(request.url, bytes):
            response.url = request.url.decode('utf-8')
        else:
            response.url = request.url

        # Give the Response some context.
        response.request = request
        response.connection = self

        return response

 def start_response(status, headers):
     response.status_code = int(status.split(' ')[0])
     response.reason = responses.get(response.status_code, 'Unknown Status Code')
     response.headers = CaseInsensitiveDict(headers)
     response.encoding = get_encoding_from_headers(response.headers)
     response.elapsed = datetime.datetime.utcnow() - start
     self._log(response)

def encoding(rsp):
    """
    encoding of Response.content.

    if Response.encoding is None, encoding will be guessed
    by header or content or chardet if avaibable.
    """
    # content is unicode
    if isinstance(rsp.content, six.text_type):
        return 'unicode'

    # Try charset from content-type
    encoding = get_encoding_from_headers(rsp.headers)
    if encoding == 'ISO-8859-1':
        encoding = None

    # Try charset from content
    if not encoding and get_encodings_from_content:
        encoding = get_encodings_from_content(rsp.content)
        encoding = encoding and encoding[0] or None

    # Fallback to auto-detected encoding.
    if not encoding and chardet is not None:
        encoding = chardet.detect(rsp.content)['encoding']

    if encoding and encoding.lower() == 'gb2312':
        encoding = 'gb18030'

    encoding = encoding or 'utf-8'
    return encoding

 def request(method, url, **kwargs):
     if 'data' in kwargs:
         kwargs['params'] = kwargs.pop('data')
     elif 'params' in kwargs and kwargs['params'] is None:
         kwargs.pop('params')
     auth = None
     if 'auth' in kwargs:
         auth = kwargs.pop('auth')
     for i in ['auth', 'allow_redirects', 'stream']:
         if i in kwargs:
             kwargs.pop(i)
     if app.app.registry.api_url in url:
         if auth:
             authorization = api.authorization
             api.authorization = ('Basic', auth)
         resp = api._gen_request(method.upper(), url, expect_errors=True, **kwargs)
         if auth:
             api.authorization = authorization
     else:
         resp = app._gen_request(method.upper(), url, expect_errors=True, **kwargs)
     response = Response()
     response.status_code = resp.status_int
     response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
     response.encoding = get_encoding_from_headers(response.headers)
     response.raw = resp
     response._content = resp.body
     response.reason = resp.status
     if isinstance(url, bytes):
         response.url = url.decode('utf-8')
     else:
         response.url = url
     response.request = resp.request
     return response

def guess_response_encoding(resp):
    '''
    Guess the content encoding of a requests response.

    Note: there's a performance issue due to chardet.
    '''
    # first try the encoding supplied by responce header and content
    encs = get_encodings_from_content(resp.content) or []
    for enc in encs:
        try:
            resp.content.decode(enc)
            LOG.info('Detected encoding %s from response content.', enc)
            return enc
        except UnicodeDecodeError:
            LOG.debug('Encoding from response content doesn\'t work.')

    enc = get_encoding_from_headers(resp.headers)
    if enc:
        try:
            resp.content.decode(enc)
            LOG.info('Detected encoding %s from response header.', enc)
            return enc
        except UnicodeDecodeError:
            LOG.debug('Encoding from response header doesn\'t work.')

    # neither encoding works, we have to go the hard way.
    start = clock()
    g = detect(resp.content)
    LOG.info('Detected encoding %s with cofidence of %g in %gs.' % (g['encoding'], g['confidence'], clock() - start))
    return g['encoding']

    def build_response(self, request, resp):
        """
        Builds a Requests' response object.  This emulates most of the logic of
        the standard fuction but deals with the lack of the ``.headers``
        property on the HTTP20Response object.
        """
        response = Response()

        response.status_code = resp.status
        response.headers = CaseInsensitiveDict(resp.getheaders())
        response.raw = resp
        response.reason = resp.reason
        response.encoding = get_encoding_from_headers(response.headers)

        extract_cookies_to_jar(response.cookies, request, response)

        if isinstance(request.url, bytes):
            response.url = request.url.decode('utf-8')
        else:
            response.url = request.url

        response.request = request
        response.connection = self

        # One last horrible patch: Requests expects its raw responses to have a
        # release_conn method, which I don't. We should monkeypatch a no-op on.
        resp.release_conn = lambda: None

        return response

def create_response(request, **kwargs):
    """
    :param int status_code: The status code to return upon a successful
        match. Defaults to 200.
    :param HTTPResponse raw: A HTTPResponse object to return upon a
        successful match.
    :param io.IOBase body: An IO object with a read() method that can
        return a body on successful match.
    :param bytes content: A byte string to return upon a successful match.
    :param unicode text: A text string to return upon a successful match.
    :param object json: A python object to be converted to a JSON string
        and returned upon a successful match.
    :param dict headers: A dictionary object containing headers that are
        returned upon a successful match.
    :param CookieJar cookies: A cookie jar with cookies to set on the
        response.
    """
    connection = kwargs.pop('connection', _FakeConnection())

    _check_body_arguments(**kwargs)

    raw = kwargs.pop('raw', None)
    body = kwargs.pop('body', None)
    content = kwargs.pop('content', None)
    text = kwargs.pop('text', None)
    json = kwargs.pop('json', None)
    headers = kwargs.pop('headers', {})
    encoding = None

    if content is not None and not isinstance(content, six.binary_type):
        raise TypeError('Content should be binary data')
    if text is not None and not isinstance(text, six.string_types):
        raise TypeError('Text should be string data')

    if json is not None:
        text = jsonutils.dumps(json)
    if text is not None:
        encoding = get_encoding_from_headers(headers) or 'utf-8'
        content = text.encode(encoding)
    if content is not None:
        body = _IOReader(content)
    if not raw:
        raw = HTTPResponse(status=kwargs.get('status_code', _DEFAULT_STATUS),
                           headers=headers,
                           reason=kwargs.get('reason'),
                           body=body or _IOReader(six.b('')),
                           decode_content=False,
                           preload_content=False,
                           original_response=compat._fake_http_response)

    response = _http_adapter.build_response(request, raw)
    response.connection = connection

    if encoding and not response.encoding:
        response.encoding = encoding

    _extract_cookies(request, response, kwargs.get('cookies'))

    return response

 def __init__(self, response, method):
     self._response = response
     self._method = method
     self.status_code = response.code
     self.headers = CaseInsensitiveDict((
         (header, ', '.join(values)) for header, values in
         response.headers.getAllRawHeaders()))
     self.encoding = get_encoding_from_headers(self.headers) or 'ISO-8859-1'
     self._waiting_for_content = []

def get_unicode_from_response(response):
    """Return the requested content back in unicode.

    This will first attempt to retrieve the encoding from the response
    headers. If that fails, it will use
    :func:`requests_toolbelt.utils.deprecated.get_encodings_from_content`
    to determine encodings from HTML elements.

    .. code-block:: python

        import requests
        from requests_toolbelt.utils import deprecated

        r = requests.get(url)
        text = deprecated.get_unicode_from_response(r)

    :param response: Response object to get unicode content from.
    :type response: requests.models.Response
    """
    tried_encodings = set()

    # Try charset from content-type
    encoding = utils.get_encoding_from_headers(response.headers)

    if encoding:
        try:
            return str(response.content, encoding)
        except UnicodeError:
            tried_encodings.add(encoding.lower())

    encodings = get_encodings_from_content(response.content)

    for _encoding in encodings:
        _encoding = _encoding.lower()
        if _encoding in tried_encodings:
            continue
        try:
            return str(response.content, _encoding)
        except UnicodeError:
            tried_encodings.add(_encoding)

    # Fall back:
    if encoding:
        try:
            return str(response.content, encoding, errors='replace')
        except TypeError:
            pass
    return response.text