本文整理匯總了Python中future.backports.urllib.parse方法的典型用法代碼示例。如果您正苦於以下問題：Python urllib.parse方法的具體用法？Python urllib.parse怎麽用？Python urllib.parse使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類future.backports.urllib的用法示例。
在下文中一共展示了urllib.parse方法的4個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: can_fetch
# 需要導入模塊: from future.backports import urllib [as 別名]
# 或者: from future.backports.urllib import parse [as 別名]
def can_fetch(self, useragent, url):
"""using the parsed robots.txt decide if useragent can fetch url"""
if self.disallow_all:
return False
if self.allow_all:
return True
# search for given user agent matches
# the first match counts
parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
url = urllib.parse.urlunparse(('','',parsed_url.path,
parsed_url.params,parsed_url.query, parsed_url.fragment))
url = urllib.parse.quote(url)
if not url:
url = "/"
for entry in self.entries:
if entry.applies_to(useragent):
return entry.allowance(url)
# try the default entry last
if self.default_entry:
return self.default_entry.allowance(url)
# agent not found ==> access granted
return True 示例2: set_url
# 需要導入模塊: from future.backports import urllib [as 別名]
# 或者: from future.backports.urllib import parse [as 別名]
def set_url(self, url):
"""Sets the URL referring to a robots.txt file."""
self.url = url
self.host, self.path = urllib.parse.urlparse(url)[1:3] 示例3: read
# 需要導入模塊: from future.backports import urllib [as 別名]
# 或者: from future.backports.urllib import parse [as 別名]
def read(self):
"""Reads the robots.txt URL and feeds it to the parser."""
try:
f = urllib.request.urlopen(self.url)
except urllib.error.HTTPError as err:
if err.code in (401, 403):
self.disallow_all = True
elif err.code >= 400:
self.allow_all = True
else:
raw = f.read()
self.parse(raw.decode("utf-8").splitlines()) 示例4: __init__
# 需要導入模塊: from future.backports import urllib [as 別名]
# 或者: from future.backports.urllib import parse [as 別名]
def __init__(self, path, allowance):
if path == '' and not allowance:
# an empty value means allow all
allowance = True
self.path = urllib.parse.quote(path)
self.allowance = allowance