Python reqser.request_to_dict方法代码示例

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def publish_links(self, requests_new, urls_other):
        # Serialize requests_new
        reqds_new = []
        for request_new in requests_new:
            try:
                reqd_new = request_to_dict(request_new, self._spider)
                reqds_new.append(reqd_new)
            except ValueError as e:
                logger.error(
                    u'Unable to serialize request: %(request)s - reason: %(reason)s',
                     {'request': request_new, 'reason': e},
                     exc_info=True, extra={'spider': self._spider}
                )

        # Send to queue
        if len(urls_other) <= 0 and len(reqds_new) <= 0:
            return

        payload = {
            'requests_new': reqds_new,
            'urls_other': urls_other,
        }
        self._queue_links.publish(payload) 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def _encode_request(self, request):
        """Encode a request object"""
        return pickle.dumps(request_to_dict(request, self.spider), protocol=-1) 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def process_request(self, request, spider):
        """ Login if we are not logged in yet.
        """
        if '_autologin' in request.meta or request.meta.get('skip_autologin'):
            returnValue(None)
        yield self._ensure_login(request, spider)
        self.stats.set_value('autologin/logged_in', self.logged_in)
        if self.skipped:
            request.meta['autologin_active'] = False
            returnValue(None)
        elif self.logged_in:
            request.meta['autologin_active'] = True
            logout_url = request.meta.get(
                'autologin_logout_url', self.logout_url)
            if logout_url and logout_url in request.url:
                logger.debug('Ignoring logout request %s', request.url)
                raise IgnoreRequest
            # Save original request to be able to retry it in case of logout
            req_copy = request.replace(meta=deepcopy(request.meta))
            request.meta['_autologin'] = autologin_meta = {}
            try:
                autologin_meta['request'] = request_to_dict(
                    req_copy, spider=spider)
            except ValueError:
                # Serialization failed, but it might be ok if we do not persist
                # requests, so store the request itself here.
                autologin_meta['request'] = req_copy
            # TODO - it should be possible to put auth cookies into the
            # cookiejar in process_response (but also check non-splash)
            if self.auth_cookies:
                request.cookies = self.auth_cookies
                autologin_meta['cookie_dict'] = {
                    c['name']: c['value'] for c in self.auth_cookies} 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def retry_request(self, request):
        try:
            reqd = request_to_dict(request, self._spider)
            self._queue_requests.retry(reqd, reqd['priority'])
        except ValueError as e:
            logger.error(
                u'Unable to serialize request: %(request)s - reason: %(reason)s',
                {'request': request, 'reason': e},
                exc_info=True, extra={'spider': self._spider}
            ) 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def publish_error(self, request, exception):
        error = {
            'request': request_to_dict(request, self._spider),
            'cls': self._get_exception_class(exception),
            'msg': str(exception)
        }
        self._queue_errors.publish(error) 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def response_to_dict(response):
    return {
        'cls': '{r.__module__}.{r.__class__.__name__}'.format(r=response),
        'encoding': response.encoding,

        'request': request_to_dict(response.request),

        'url': response.url,
        'status': response.status,
        'headers': dict(response.headers),
        'meta': response.meta,

        'body': response.body,
    } 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def _encode_request(self, request):
        """Encode a request object"""
        obj = request_to_dict(request, self.spider)
        return self.serializer.dumps(obj) 

# 需要导入模块: from scrapy.utils import reqser [as 别名]
# 或者: from scrapy.utils.reqser import request_to_dict [as 别名]
def _encode_request(self, request):
        return pickle.dumps(request_to_dict(request, self.spider), protocol=-1)