本文整理汇总了Python中scrapy.utils.response.response_status_message方法的典型用法代码示例。如果您正苦于以下问题：Python response.response_status_message方法的具体用法？Python response.response_status_message怎么用？Python response.response_status_message使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.utils.response的用法示例。
在下文中一共展示了response.response_status_message方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效
logger.warning("One Cookie need to be updating...")
update_cookie(request.meta['accountText'], self.rconn, spider.name)
elif "weibo.cn/security" in redirect_url: # 账号被限
logger.warning("One Account is locked! Remove it!")
remove_cookie(request.meta["accountText"], self.rconn, spider.name)
elif "weibo.cn/pub" in redirect_url:
logger.warning(
"Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
logging.warning(e.message)
raise IgnoreRequest 示例2: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
request.headers['User-Agent'] = self.get_ua()
return self._retry(request, reason, spider) or response
return response 示例3: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
# 删除该代理
self.delete_proxy(request.meta.get('proxy', False))
print('返回值异常, 进行重试...')
return self._retry(request, reason, spider) or response
return response 示例4: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
return response 示例5: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
return response 示例6: process_response
# 需要导入模块: from scrapy.utils import response [as 别名]
# 或者: from scrapy.utils.response import response_status_message [as 别名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
logging.info("response.status:" + str(response.status))
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
self.delete_proxy(request.meta.get('proxy', False))
time.sleep(random.randint(10, 12))
self.logger.warning('返回值异常, 进行重试...')
return self._retry(request, reason, spider) or response
return response