本文整理汇总了Python中scrapy.utils.reqser.request_from_dict方法的典型用法代码示例。如果您正苦于以下问题：Python reqser.request_from_dict方法的具体用法？Python reqser.request_from_dict怎么用？Python reqser.request_from_dict使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.utils.reqser的用法示例。
在下文中一共展示了reqser.request_from_dict方法的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: response_from_dict
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def response_from_dict(responsed):
respcls = load_object(responsed['cls'])
request = request_from_dict(responsed['request'])
response = respcls(
encoding=responsed['encoding'],
request=request,
url=responsed['url'],
status=responsed['status'],
headers=responsed['headers'],
body=responsed['body'],
)
response.meta.update(responsed['meta'])
return response 示例2: _decode_request
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def _decode_request(self, encoded_request):
"""Decode an request previously encoded"""
return request_from_dict(pickle.loads(encoded_request), self.spider) 示例3: _decode_request
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def _decode_request(self, encoded_request):
"""Decode a request previously encoded"""
return request_from_dict(pickle.loads(encoded_request), self.spider) 示例4: process_response
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def process_response(self, request, response, spider):
""" If we were logged out, login again and retry request.
"""
if request.meta.get('_autologin') and self.is_logout(response):
autologin_meta = request.meta['_autologin']
if isinstance(autologin_meta['request'], dict):
retryreq = request_from_dict(autologin_meta['request'], spider)
else:
retryreq = autologin_meta['request'].copy()
retryreq.dont_filter = True
logger.debug(
'Logout at %s: %s', retryreq.url, _response_cookies(response))
if self.logged_in:
# We could have already done relogin after initial logout
if any(autologin_meta['cookie_dict'].get(c['name']) !=
c['value'] for c in self.auth_cookies):
logger.debug('Request was stale, will retry %s', retryreq)
else:
self.logged_in = False
# It's better to re-login straight away
yield self._ensure_login(retryreq, spider)
logout_count = retryreq.meta['autologin_logout_count'] = (
retryreq.meta.get('autologin_logout_count', 0) + 1)
if logout_count >= self.max_logout_count:
logger.debug('Max logouts exceeded, will not retry %s',
retryreq)
raise IgnoreRequest
else:
logger.debug(
'Request caused log out (%d), still retrying %s',
logout_count, retryreq)
returnValue(retryreq)
returnValue(response) 示例5: get_next_request
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def get_next_request(self):
reqd = self._queue_requests.get()
if not reqd:
return
return request_from_dict(reqd, self._spider) 示例6: _decode_request
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def _decode_request(self, encoded_request):
"""Decode an request previously encoded"""
obj = self.serializer.loads(encoded_request)
return request_from_dict(obj, self.spider) 示例7: _decode_request
# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_from_dict [as 别名]
def _decode_request(self, encoded_request):
return request_from_dict(pickle.loads(encoded_request), self.spider)