本文整理汇总了Python中scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware类的典型用法代码示例。如果您正苦于以下问题：Python HttpErrorMiddleware类的具体用法？Python HttpErrorMiddleware怎么用？Python HttpErrorMiddleware使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了HttpErrorMiddleware类的9个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestHttpErrorMiddlewareHandleAll
class TestHttpErrorMiddlewareHandleAll(TestCase):
def setUp(self):
self.spider = BaseSpider("foo")
self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
self.req = Request("http://scrapytest.org")
self.res200 = Response("http://scrapytest.org", status=200)
self.res200.request = self.req
self.res404 = Response("http://scrapytest.org", status=404)
self.res404.request = self.req
self.res402 = Response("http://scrapytest.org", status=402)
self.res402.request = self.req
def test_process_spider_input(self):
self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
res402.request = request
self.assertEquals(None, self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)示例2: TestHttpErrorMiddleware
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider("foo")
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request("http://scrapytest.org")
self.res200 = Response("http://scrapytest.org", status=200)
self.res200.request = self.req
self.res404 = Response("http://scrapytest.org", status=404)
self.res404.request = self.req
def test_process_spider_input(self):
self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
self.assertEquals(None, self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))示例3: TestHttpErrorMiddleware
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertEquals(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEquals([],
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEquals(None,
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEquals(None,
self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEquals(None,
self.mw.process_spider_input(self.res404, self.spider))示例4: TestHttpErrorMiddlewareHandleAll
class TestHttpErrorMiddlewareHandleAll(TestCase):
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404, self.res402 = _responses(
self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEquals(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertEquals(None,
self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
res402.request = request
self.assertEquals(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)示例5: TestHttpErrorMiddleware
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider()
self.mw = HttpErrorMiddleware()
self.req = Request('http://scrapytest.org')
self.res200 = Response('http://scrapytest.org', status=200)
self.res200.request = self.req
self.res404 = Response('http://scrapytest.org', status=404)
self.res404.request = self.req
def test_process_spider_input(self):
self.assertEquals(self.mw.process_spider_input(self.res200, self.spider),
None)
self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
[])
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEquals(self.mw.process_spider_input(res, self.spider),
None)
self.spider.handle_httpstatus_list = [404]
self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
None)示例6: setUp
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(
Settings({'HTTPERROR_ALLOWED_CODES': (402,)}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404, self.res402 = _responses(
self.req, [200, 404, 402])示例7: setUp
def setUp(self):
self.spider = BaseSpider("foo")
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request("http://scrapytest.org")
self.res200 = Response("http://scrapytest.org", status=200)
self.res200.request = self.req
self.res404 = Response("http://scrapytest.org", status=404)
self.res404.request = self.req示例8: setUp
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request('http://scrapytest.org')
self.res200 = Response('http://scrapytest.org', status=200)
self.res200.request = self.req
self.res404 = Response('http://scrapytest.org', status=404)
self.res404.request = self.req示例9: TestHttpErrorMiddlewareSettings
class TestHttpErrorMiddlewareSettings(TestCase):
"""Similar test, but with settings"""
def setUp(self):
self.spider = Spider('foo')
self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)}))
self.req = Request('http://scrapytest.org')
self.res200 = Response('http://scrapytest.org', status=200)
self.res200.request = self.req
self.res404 = Response('http://scrapytest.org', status=404)
self.res404.request = self.req
self.res402 = Response('http://scrapytest.org', status=402)
self.res402.request = self.req
def test_process_spider_input(self):
self.assertEquals(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertEquals(None,
self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
res402.request = request
self.assertEquals(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertEquals(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res402, self.spider)