Python urllib.addinfourl方法代码示例

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def probe_html5(self, result):

        class NoRedirectHandler(urllib2.HTTPRedirectHandler):

            def http_error_302(self, req, fp, code, msg, headers):
                infourl = urllib.addinfourl(fp, headers, req.get_full_url())
                infourl.status = code
                infourl.code = code
                return infourl
            http_error_300 = http_error_302
            http_error_301 = http_error_302
            http_error_303 = http_error_302
            http_error_307 = http_error_302

        opener = urllib2.build_opener(NoRedirectHandler())
        urllib2.install_opener(opener)

        r = urllib2.urlopen(urllib2.Request(result['url'], headers=result['headers']))
        if r.code == 200:
            result['url'] = r.read()
        return result 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def _normalize_urllib_response(response, request=None):
    if six.PY2:
        if not isinstance(response, urllib.addinfourl):
            raise TypeError("Cannot normalize this response object")
    else:
        if not isinstance(response, http.client.HTTPResponse):
            raise TypeError("Cannot normalize this response object")

    url = response.url
    status_code = response.getcode()
    content_type = response.headers.get('Content-Type')

    return Response(
        request=request,
        content=response.read(),
        url=url,
        status_code=status_code,
        content_type=content_type,
        response=response,
    ) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def get_response(self):
        """Returns a copy of the current response."""
        return urllib.addinfourl(StringIO(self.data), self._response.info(), self._response.geturl()) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def _make_response(self, result, url):
        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])
        headers = httplib.HTTPMessage(StringIO(data))
        response = urllib.addinfourl(StringIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def wsopen(self, url, post, **params):
                noparam = params.pop('noparam',False)
                if noparam:
                        params = {}
                else:
                        if self.user is not None:
                                params['user'] = self.user
                        if self.password is not None:
                                params.pop('hmac', None)
                                HMAC=hmac.new(self.password)
                                for k,v in sorted(params.items()):
                                        HMAC.update("%s=%s" % (k,v))
                                params.update({'hmac':HMAC.hexdigest()})
                query = urllib.urlencode(params)
                if post:
                        body = query
                elif query:
                        url = "{}?{}".format(url, query)

                if self.debug:
                        if post:
                                print("POST:\n{}\n{!r}\n".format(url, body), file=sys.stderr)
                        else:
                                print("GET:\n{}\n".format(url), file=sys.stderr)

                class URLopener(urllib.FancyURLopener):
                        def http_error_default(self, url, fp, errcode, errmsg, headers):
                                return urllib.addinfourl(fp, headers, "http:" + url, errcode)
                try:
                        urllib._urlopener = URLopener()
                        if post:
                                resp = urllib.urlopen(url, body)
                        else:
                                resp = urllib.urlopen(url)
                except IOError as e:
                        raise WSError(url, msg=e)
                if self.debug:
                        print("RESPONSE:\n{}\n{}".format(resp.getcode(), resp.info()), file=sys.stderr)
                if resp.getcode() != 200:
                        raise WSError(url, resp.getcode(), resp.read())
                return resp 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def get_response(self):
        """Returns a copy of the current response."""
        return addinfourl(BytesIO(self.data), self._response.info(), self._response.geturl()) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def _make_response(self, result, url):

        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])

        if PY2:
            headers = HTTPMessage(BytesIO(data))
        else:
            import email
            headers = email.message_from_string(data)

        response = addinfourl(BytesIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def data_open(self, req):
            # data URLs as specified in RFC 2397.
            #
            # ignores POSTed data
            #
            # syntax:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            url = req.get_full_url()

            scheme, data = url.split(':', 1)
            mediatype, data = data.split(',', 1)

            # even base64 encoded data URLs might be quoted so unquote in any case:
            data = compat_urllib_parse_unquote_to_bytes(data)
            if mediatype.endswith(';base64'):
                data = binascii.a2b_base64(data)
                mediatype = mediatype[:-7]

            if not mediatype:
                mediatype = 'text/plain;charset=US-ASCII'

            headers = email.message_from_string(
                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))

            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_206(self, req, fp, code, msg, hdrs):
        # 206 Partial Content Response
        r = urllib.addinfourl(fp, hdrs, req.get_full_url())
        r.code = code
        r.msg = msg
        return r 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_302(self, req, fp, code, msg, headers):
            infourl = urllib.addinfourl(fp, headers, req.get_full_url())
            infourl.status = code
            infourl.code = code
            return infourl 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, headers["Location"])
        infourl.status = code
        infourl.code = code
        return infourl 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_default(self, req, fp, code, msg, headers):
        if ((code / 100) == 3) and (code != 304):
            return self.http_error_302(req, fp, code, msg, headers)
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        return infourl 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_302(self, req, fp, code, msg, headers):
        if headers.dict.has_key('location'):
            infourl = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
        else:
            infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        if not hasattr(infourl, 'status'):
            infourl.status = code
        return infourl 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import addinfourl [as 别名]
def http_error_301(self, req, fp, code, msg, headers):
        if headers.dict.has_key('location'):
            infourl = urllib2.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, headers)
        else:
            infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        if not hasattr(infourl, 'status'):
            infourl.status = code
        return infourl