本文整理汇总了Python中scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware.process_spider_exception方法的典型用法代码示例。如果您正苦于以下问题：Python HttpErrorMiddleware.process_spider_exception方法的具体用法？Python HttpErrorMiddleware.process_spider_exception怎么用？Python HttpErrorMiddleware.process_spider_exception使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware的用法示例。
在下文中一共展示了HttpErrorMiddleware.process_spider_exception方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestHttpErrorMiddleware
# 需要导入模块: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 别名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_exception [as 别名]
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider("foo")
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request("http://scrapytest.org")
self.res200 = Response("http://scrapytest.org", status=200)
self.res200.request = self.req
self.res404 = Response("http://scrapytest.org", status=404)
self.res404.request = self.req
def test_process_spider_input(self):
self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
self.assertEquals(None, self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))示例2: TestHttpErrorMiddleware
# 需要导入模块: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 别名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_exception [as 别名]
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertEquals(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEquals([],
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEquals(None,
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEquals(None,
self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEquals(None,
self.mw.process_spider_input(self.res404, self.spider))