本文整理匯總了Python中scrapy.utils.response.response_status_message方法的典型用法代碼示例。如果您正苦於以下問題：Python response.response_status_message方法的具體用法？Python response.response_status_message怎麽用？Python response.response_status_message使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類scrapy.utils.response的用法示例。
在下文中一共展示了response.response_status_message方法的6個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效
logger.warning("One Cookie need to be updating...")
update_cookie(request.meta['accountText'], self.rconn, spider.name)
elif "weibo.cn/security" in redirect_url: # 賬號被限
logger.warning("One Account is locked! Remove it!")
remove_cookie(request.meta["accountText"], self.rconn, spider.name)
elif "weibo.cn/pub" in redirect_url:
logger.warning(
"Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重試
except Exception, e:
logging.warning(e.message)
raise IgnoreRequest 示例2: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
request.headers['User-Agent'] = self.get_ua()
return self._retry(request, reason, spider) or response
return response 示例3: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
# 刪除該代理
self.delete_proxy(request.meta.get('proxy', False))
print('返回值異常, 進行重試...')
return self._retry(request, reason, spider) or response
return response 示例4: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
return response 示例5: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
return response 示例6: process_response
# 需要導入模塊: from scrapy.utils import response [as 別名]
# 或者: from scrapy.utils.response import response_status_message [as 別名]
def process_response(self, request, response, spider):
if request.meta.get('dont_retry', False):
return response
logging.info("response.status:" + str(response.status))
if response.status in self.retry_http_codes:
reason = response_status_message(response.status)
self.delete_proxy(request.meta.get('proxy', False))
time.sleep(random.randint(10, 12))
self.logger.warning('返回值異常, 進行重試...')
return self._retry(request, reason, spider) or response
return response