Python Request.header_items方法代码示例

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
class WEBService(object):

    """Class to query web services."""

    def __init__(self, url, user_agent=UA, values=None, appheaders=None):
        """Initialize main properties."""
        self._url = url
        # headers to accept gzipped content
        headers = {'Accept-Encoding': 'gzip', 'User-Agent': user_agent}
        # add more user provided headers
        if appheaders:  # pragma: no cover
            headers.update(appheaders)
        # if 'data' it does a PUT request (data must be urlencoded)
        data = urlencode(values) if values else None
        self._request = Request(url, data, headers=headers)
        self.response = None

    def _response(self):
        """Check errors on response."""
        try:
            self.response = urlopen(self._request)
            LOGGER.debug('Request headers:\n%s', self._request.header_items())
        except HTTPError as e:  # pragma: no cover
            LOGGER.critical('ISBNLibHTTPError for %s with code %s [%s]',
                            self._url, e.code, e.msg)
            if e.code in (401, 403, 429):
                raise ISBNLibHTTPError('%s Are you making many requests?'
                                       % e.code)
            if e.code in (502, 504):
                raise ISBNLibHTTPError('%s Service temporarily unavailable!'
                                       % e.code)
            raise ISBNLibHTTPError('(%s) %s' % (e.code, e.msg))
        except URLError as e:   # pragma: no cover
            LOGGER.critical('ISBNLibURLError for %s with reason %s',
                            self._url, e.reason)
            raise ISBNLibURLError(e.reason)

    def data(self):
        """Return the uncompressed data."""
        self._response()
        LOGGER.debug('Response headers:\n%s', self.response.info())
        if self.response.info().get('Content-Encoding') == 'gzip':
            buf = bstream(self.response.read())
            f = gzip.GzipFile(fileobj=buf)
            data = f.read()
        else:                   # pragma: no cover
            data = self.response.read()
        return s(data)

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
def _download(url, tofile=None):
    """Download image."""
    headers = {'User-Agent': UA, 'Pragma': 'no-cache'}
    request = Request(url, headers=headers)
    try:
        response = urlopen(request)
        LOGGER.debug('Request headers:\n%s', request.header_items())
    except HTTPError as e:  # pragma: no cover
        LOGGER.critical('ISBNLibHTTPError for %s with code %s [%s]',
                        url, e.code, e.msg)
        if e.code == 403:
            # Google uses this code when the image is not
            # available for any size (should use 404),
            # but also when you are blacklisted by the service
            LOGGER.debug('Cover not available or you are making many requests')
            return True     # <-- no more attempts to download
        if e.code in (401, 429):
            raise ISBNLibHTTPError('%s Are you are making many requests?'
                                   % e.code)
        if e.code in (502, 504):
            raise ISBNLibHTTPError('%s Service temporarily unavailable!'
                                   % e.code)
        raise ISBNLibHTTPError('(%s) %s' % (e.code, e.msg))
    except URLError as e:   # pragma: no cover
        LOGGER.critical('ISBNLibURLError for %s with reason %s',
                        url, e.reason)
        raise ISBNLibURLError(e.reason)
    content = response.read()
    noimageavailable = len(content) in NOIMGSIZE
    if noimageavailable:    # pragma: no cover
        return False
    if tofile:
        try:                # pragma: no cover
            # PY2
            content_type = response.info().getheader('Content-Type')
        except:             # pragma: no cover
            # PY3
            content_type = response.getheader('Content-Type')
        _, ext = content_type.split('/')
        tofile = tofile.split('.')[0] + '.' + ext.split('-')[-1]
        with open(tofile, 'wb') as f:
            f.write(content)
    else:                   # pragma: no cover
        print(content)
    return tofile

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
    def __api_request(self, request_uri, body, headers={}):
        url = self.uri
        url += request_uri

        # body must by type 'bytes' for 3.x
        if _isunicode(body):
            body = body.encode()

        request = Request(url, body, headers)

        self._log(DEBUG1, 'URL: %s', url)
        self._log(DEBUG1, 'method: %s', request.get_method())
        self._log(DEBUG1, 'headers: %s', request.header_items())

        # XXX leaks apikey
#        self._log(DEBUG3, 'body: %s', repr(body))

        kwargs = {
            'url': request,
            }

        if self.ssl_context is not None:
            kwargs['context'] = self.ssl_context

        if self.timeout is not None:
            kwargs['timeout'] = self.timeout

        try:
            response = self._urlopen(**kwargs)
        except self._certificateerror as e:
            self._msg = 'ssl.CertificateError: %s' % e
            return False
        except (URLError, IOError) as e:
            self._log(DEBUG2, 'urlopen() exception: %s', sys.exc_info())
            self._msg = str(e)
            return False

        self.http_code = response.getcode()
        if hasattr(response, 'reason'):
            # 3.2
            self.http_reason = response.reason
        elif hasattr(response, 'msg'):
            # 2.7
            self.http_reason = response.msg

        if self.http_reason == '':
            if self.http_code in _wildfire_responses:
                self.http_reason = _wildfire_responses[self.http_code]
            elif self.http_code in responses:
                self.http_reason = responses[self.http_code]

        try:
            self._message = email.message_from_string(str(response.info()))
        except (TypeError, email.errors.MessageError) as e:
            raise PanWFapiError('email.message_from_string() %s' % e)

        self._log(DEBUG2, 'HTTP response code: %s', self.http_code)
        self._log(DEBUG2, 'HTTP response reason: %s', self.http_reason)
        self._log(DEBUG2, 'HTTP response headers:')
        self._log(DEBUG2, '%s', self._message)

        if not (200 <= self.http_code < 300):
            self._msg = 'HTTP Error %s: %s' % (self.http_code,
                                               self.http_reason)
            self.__set_response(response)
            return False

        return response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
 def wrapper(client, method, url, body=None, headers=None):
     cookiejar = LWPCookieJar()
     cookiejar._really_load(
         StringIO("#LWP-Cookies-2.0\n" + client.credential.get(field,'')),
         "cookies.txt",True,True)
     req = Request(url, body, headers or {}, method=method)
     cookiejar.clear_expired_cookies()
     cookiejar.add_cookie_header(req)
     status, headers, body = request(client,req.method,req.full_url,req.data,dict(req.header_items()))
     response = addinfourl(None, headers, req.full_url, status)
     cookiejar.extract_cookies(response,req)
     client.credential[field] = cookiejar.as_lwp_str()
     return (status, headers, body)

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
    def __api_request(self, request_uri, body, headers={}):
        url = self.uri
        url += request_uri

        self._log(DEBUG1, 'URL: %s', url)
        self._log(DEBUG1, 'headers: %s', headers)

        # body must by type 'bytes' for 3.x
        if _isunicode(body):
            body = body.encode()

        self._log(DEBUG3, 'body: %s', repr(body))

        request = Request(url, body, headers)

        self._log(DEBUG1, 'method: %s', request.get_method())
        self._log(DEBUG1, 'headers: %s', request.header_items())

        kwargs = {
            'url': request,
            }
        # Changed in version 3.2: cafile and capath were added.
        if sys.hexversion >= 0x03020000:
            kwargs['cafile'] = self.cafile
            kwargs['capath'] = self.capath
        # Changed in version 3.3: cadefault added
        if sys.hexversion >= 0x03030000:
            pass
#            kwargs['cadefault'] = True

        if self.timeout is not None:
            kwargs['timeout'] = self.timeout

        try:

            response = PanWFapi._urlopen(**kwargs)

        # invalid cafile, capath
        except (URLError, IOError) as e:
            self._log(DEBUG2, 'urlopen() exception: %s', sys.exc_info())
            self._msg = str(e)
            return False

        self.http_code = response.getcode()
        if hasattr(response, 'reason'):
            # 3.2
            self.http_reason = response.reason
        elif hasattr(response, 'msg'):
            # 2.7
            self.http_reason = response.msg

        if self.http_reason == '':
            if self.http_code in _wildfire_responses:
                self.http_reason = _wildfire_responses[self.http_code]
            elif self.http_code in responses:
                self.http_reason = responses[self.http_code]

        self._log(DEBUG2, 'HTTP response code: %s', self.http_code)
        self._log(DEBUG2, 'HTTP response reason: %s', self.http_reason)
        self._log(DEBUG2, 'HTTP response headers:')
        self._log(DEBUG2, '%s', response.info())

        if not (200 <= self.http_code < 300):
            self._msg = 'HTTP Error %s: %s' % (self.http_code,
                                               self.http_reason)
            self.__set_response(response)
            return False

        return response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
    def __api_request(self, request_uri, body, headers={}):
        url = self.uri
        url += request_uri

        self._log(DEBUG1, 'URL: %s', url)
        self._log(DEBUG1, 'headers: %s', headers)

        # body must by type 'bytes' for 3.x
        if _isunicode(body):
            body = body.encode()

        self._log(DEBUG3, 'body: %s', repr(body))

        request = Request(url, body, headers)

        self._log(DEBUG1, 'method: %s', request.get_method())
        self._log(DEBUG1, 'headers: %s', request.header_items())

        kwargs = {
            'url': request,
            }

        if self.ssl_context is not None:
            kwargs['context'] = self.ssl_context

        if self.timeout is not None:
            kwargs['timeout'] = self.timeout

        try:
            response = self._urlopen(**kwargs)
        except (URLError, IOError) as e:
            self._log(DEBUG2, 'urlopen() exception: %s', sys.exc_info())
            self._msg = str(e)
            return False

        self.http_code = response.getcode()
        if hasattr(response, 'reason'):
            # 3.2
            self.http_reason = response.reason
        elif hasattr(response, 'msg'):
            # 2.7
            self.http_reason = response.msg

        if self.http_reason == '':
            if self.http_code in _wildfire_responses:
                self.http_reason = _wildfire_responses[self.http_code]
            elif self.http_code in responses:
                self.http_reason = responses[self.http_code]

        self._log(DEBUG2, 'HTTP response code: %s', self.http_code)
        self._log(DEBUG2, 'HTTP response reason: %s', self.http_reason)
        self._log(DEBUG2, 'HTTP response headers:')
        self._log(DEBUG2, '%s', response.info())

        if not (200 <= self.http_code < 300):
            self._msg = 'HTTP Error %s: %s' % (self.http_code,
                                               self.http_reason)
            self.__set_response(response)
            return False

        return response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
    def __api_request(self, request_uri, body, headers={}):
        url = self.uri
        url += request_uri

        self._log(DEBUG1, "URL: %s", url)
        self._log(DEBUG1, "headers: %s", headers)

        # body must by type 'bytes' for 3.x
        if _isunicode(body):
            body = body.encode()

        self._log(DEBUG3, "body: %s", repr(body))

        request = Request(url, body, headers)

        self._log(DEBUG1, "method: %s", request.get_method())
        self._log(DEBUG1, "headers: %s", request.header_items())

        kwargs = {"url": request}

        if self.ssl_context is not None:
            kwargs["context"] = self.ssl_context

        if self.timeout is not None:
            kwargs["timeout"] = self.timeout

        try:
            response = self._urlopen(**kwargs)
        except (URLError, IOError) as e:
            self._log(DEBUG2, "urlopen() exception: %s", sys.exc_info())
            self._msg = str(e)
            return False

        self.http_code = response.getcode()
        if hasattr(response, "reason"):
            # 3.2
            self.http_reason = response.reason
        elif hasattr(response, "msg"):
            # 2.7
            self.http_reason = response.msg

        if self.http_reason == "":
            if self.http_code in _wildfire_responses:
                self.http_reason = _wildfire_responses[self.http_code]
            elif self.http_code in responses:
                self.http_reason = responses[self.http_code]

        try:
            self._message = email.message_from_string(str(response.info()))
        except (TypeError, email.errors.MessageError) as e:
            raise PanWFapiError("email.message_from_string() %s" % e)

        self._log(DEBUG2, "HTTP response code: %s", self.http_code)
        self._log(DEBUG2, "HTTP response reason: %s", self.http_reason)
        self._log(DEBUG2, "HTTP response headers:")
        self._log(DEBUG2, "%s", self._message)

        if not (200 <= self.http_code < 300):
            self._msg = "HTTP Error %s: %s" % (self.http_code, self.http_reason)
            self.__set_response(response)
            return False

        return response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
    def __api_request(self, request_uri, body, headers={}):
        url = self.uri
        url += request_uri

        if self.debug1:
            print("URL:", url, file=sys.stderr)
            print("headers:", headers, file=sys.stderr)

        # body must by type 'bytes' for 3.x
        if _isunicode(body):
            body = body.encode()

        if self.debug3:
            print("body:", repr(body), file=sys.stderr)

        request = Request(url, body, headers)

        if self.debug1:
            print("method:", request.get_method(), file=sys.stderr)
            print("headers:", request.header_items(), file=sys.stderr)

        kwargs = {"url": request}
        # Changed in version 3.2: cafile and capath were added.
        if sys.hexversion >= 0x03020000:
            kwargs["cafile"] = self.cafile
            kwargs["capath"] = self.capath
        # Changed in version 3.3: cadefault added
        if sys.hexversion >= 0x03030000:
            pass
        #            kwargs['cadefault'] = True

        if self.timeout is not None:
            kwargs["timeout"] = self.timeout

        # override HTTPError for (not 200 <= code 300) and handle below
        def http_response(request, response):
            return response

        http_error_processor = HTTPErrorProcessor()
        http_error_processor.https_response = http_response
        opener = build_opener(http_error_processor)
        # install so we can use **kwargs
        install_opener(opener)

        try:
            response = urlopen(**kwargs)

        # XXX handle httplib.BadStatusLine when http to port 443
        except URLError as e:
            self._msg = str(e)
            return False
        # invalid cafile, capath
        except IOError as e:
            self._msg = str(e)
            return False

        self.http_code = response.getcode()
        if hasattr(response, "reason"):
            # 3.2
            self.http_reason = response.reason
        elif hasattr(response, "msg"):
            # 2.7
            self.http_reason = response.msg
        elif self.http_code in responses:
            self.http_reason = responses[self.http_code]

        if self.debug2:
            print("HTTP response code:", self.http_code, file=sys.stderr)
            print("HTTP response reason:", self.http_reason, file=sys.stderr)
            print("HTTP response headers:", file=sys.stderr)
            print(response.info(), file=sys.stderr)

        if not (200 <= self.http_code < 300):
            self._msg = "HTTP Error %s: %s" % (self.http_code, self.http_reason)
            self.__set_response(response)
            return False

        return response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import header_items [as 别名]
def grabber(url, header=None, *, referer=None, cookie=None, raw=False, errorlog=None):
	"""Request url, return text or bytes of the content."""

	url = safeurl(url)

	print("[grabber]", url, "\n")

	# Build request
	request = Request(url)

	# Build header
	if header is None:
		header = {}

	# Build default header
	for key in default_header:
		if key not in header:
			header[key] = default_header[key]

	# Referer
	if referer:
		header["Referer"] = referer

	# Handle cookie
	if request.host not in cookiejar:
		cookiejar[request.host] = SimpleCookie()

	jar = cookiejar[request.host]

	if cookie:
		jar.load(cookie)

	if "Cookie" in header:
		jar.load(header["Cookie"])

	if jar:
		header["Cookie"] = "; ".join([key + "=" + c.value for key, c in jar.items()])

	header = safeheader(header)
	for key, value in header.items():
		request.add_header(key, value)

	response = urlopen(request, timeout=20)

	jar.load(response.getheader("Set-Cookie", ""))
	if cookie is not None:
		for key, c in jar.items():
			cookie[key] = c.value

	b = response.read()

	# decompress gziped data
	if response.getheader("Content-Encoding") == "gzip":
		b = decompress(b)

	if raw:
		content = b

	else:
		# find html defined encoding
		s = b.decode("utf-8", "replace")
		match = search(r"charset=[\"']?([^\"'>]+)", s)
		if match:
			s = b.decode(match.group(1), "replace")
		content = s

	if errorlog or setting.getboolean("errorlog"):
		log_object = (
			url,
			request.header_items(),
			response.getheaders()
		)
		if not errorlog:
			errorlog = ""
		from pprint import pformat
		content_write("~/comiccrawler/grabber.log", pformat(log_object) + "\n\n", append=True)

	return content