本文整理汇总了Python中future.backports.urllib.parse方法的典型用法代码示例。如果您正苦于以下问题：Python urllib.parse方法的具体用法？Python urllib.parse怎么用？Python urllib.parse使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类future.backports.urllib的用法示例。
在下文中一共展示了urllib.parse方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: can_fetch
# 需要导入模块: from future.backports import urllib [as 别名]
# 或者: from future.backports.urllib import parse [as 别名]
def can_fetch(self, useragent, url):
"""using the parsed robots.txt decide if useragent can fetch url"""
if self.disallow_all:
return False
if self.allow_all:
return True
# search for given user agent matches
# the first match counts
parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
url = urllib.parse.urlunparse(('','',parsed_url.path,
parsed_url.params,parsed_url.query, parsed_url.fragment))
url = urllib.parse.quote(url)
if not url:
url = "/"
for entry in self.entries:
if entry.applies_to(useragent):
return entry.allowance(url)
# try the default entry last
if self.default_entry:
return self.default_entry.allowance(url)
# agent not found ==> access granted
return True 示例2: set_url
# 需要导入模块: from future.backports import urllib [as 别名]
# 或者: from future.backports.urllib import parse [as 别名]
def set_url(self, url):
"""Sets the URL referring to a robots.txt file."""
self.url = url
self.host, self.path = urllib.parse.urlparse(url)[1:3] 示例3: read
# 需要导入模块: from future.backports import urllib [as 别名]
# 或者: from future.backports.urllib import parse [as 别名]
def read(self):
"""Reads the robots.txt URL and feeds it to the parser."""
try:
f = urllib.request.urlopen(self.url)
except urllib.error.HTTPError as err:
if err.code in (401, 403):
self.disallow_all = True
elif err.code >= 400:
self.allow_all = True
else:
raw = f.read()
self.parse(raw.decode("utf-8").splitlines()) 示例4: __init__
# 需要导入模块: from future.backports import urllib [as 别名]
# 或者: from future.backports.urllib import parse [as 别名]
def __init__(self, path, allowance):
if path == '' and not allowance:
# an empty value means allow all
allowance = True
self.path = urllib.parse.quote(path)
self.allowance = allowance