當前位置: 首頁>>代碼示例>>Python>>正文

Python HttpErrorMiddleware.process_spider_input方法代碼示例

本文整理匯總了Python中scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware.process_spider_input方法的典型用法代碼示例。如果您正苦於以下問題：Python HttpErrorMiddleware.process_spider_input方法的具體用法？Python HttpErrorMiddleware.process_spider_input怎麽用？Python HttpErrorMiddleware.process_spider_input使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware的用法示例。


在下文中一共展示了HttpErrorMiddleware.process_spider_input方法的6個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。

示例1: TestHttpErrorMiddleware

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddleware(TestCase):
    def setUp(self):
        self.spider = BaseSpider("foo")
        self.mw = HttpErrorMiddleware(Settings({}))
        self.req = Request("http://scrapytest.org")

        self.res200 = Response("http://scrapytest.org", status=200)
        self.res200.request = self.req
        self.res404 = Response("http://scrapytest.org", status=404)
        self.res404.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)

    def test_process_spider_exception(self):
        self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
        self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider))

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        self.assertEquals(None, self.mw.process_spider_input(res, self.spider))

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))
開發者ID:rom232，項目名稱:scrapy，代碼行數:28，代碼來源:test_spidermiddleware_httperror.py

示例2: TestHttpErrorMiddlewareHandleAll

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddlewareHandleAll(TestCase):
    def setUp(self):
        self.spider = BaseSpider("foo")
        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
        self.req = Request("http://scrapytest.org")

        self.res200 = Response("http://scrapytest.org", status=200)
        self.res200.request = self.req
        self.res404 = Response("http://scrapytest.org", status=404)
        self.res404.request = self.req
        self.res402 = Response("http://scrapytest.org", status=402)
        self.res402.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))

    def test_meta_overrides_settings(self):
        request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEquals(None, self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
開發者ID:rom232，項目名稱:scrapy，代碼行數:28，代碼來源:test_spidermiddleware_httperror.py

示例3: TestHttpErrorMiddleware

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddleware(TestCase):

    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({}))
        self.req = Request('http://scrapytest.org')
        self.res200, self.res404 = _responses(self.req, [200, 404])

    def test_process_spider_input(self):
        self.assertEquals(None,
                          self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError,
                          self.mw.process_spider_input, self.res404, self.spider)

    def test_process_spider_exception(self):
        self.assertEquals([],
                          self.mw.process_spider_exception(self.res404,
                                                           HttpError(self.res404), self.spider))
        self.assertEquals(None,
                          self.mw.process_spider_exception(self.res404,
                                                           Exception(), self.spider))

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})
        self.assertEquals(None,
                          self.mw.process_spider_input(res, self.spider))

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None,
                          self.mw.process_spider_input(self.res404, self.spider))
開發者ID:pyarnold，項目名稱:scrapy，代碼行數:34，代碼來源:test_spidermiddleware_httperror.py

示例4: TestHttpErrorMiddlewareHandleAll

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddlewareHandleAll(TestCase):

    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
        self.req = Request('http://scrapytest.org')
        self.res200, self.res404, self.res402 = _responses(
            self.req, [200, 404, 402])

    def test_process_spider_input(self):
        self.assertEquals(None,
                          self.mw.process_spider_input(self.res200, self.spider))
        self.assertEquals(None,
                          self.mw.process_spider_input(self.res404, self.spider))

    def test_meta_overrides_settings(self):
        request = Request('http://scrapytest.org',
                          meta={'handle_httpstatus_list': [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEquals(None,
                          self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError,
                          self.mw.process_spider_input, res402, self.spider)
開發者ID:pyarnold，項目名稱:scrapy，代碼行數:29，代碼來源:test_spidermiddleware_httperror.py

示例5: TestHttpErrorMiddleware

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddleware(TestCase):

    def setUp(self):
        self.spider = BaseSpider()
        self.mw = HttpErrorMiddleware()
        self.req = Request('http://scrapytest.org')

        self.res200 = Response('http://scrapytest.org', status=200)
        self.res200.request = self.req
        self.res404 = Response('http://scrapytest.org', status=404)
        self.res404.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(self.mw.process_spider_input(self.res200, self.spider),
                          None)

        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
                          [])

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})

        self.assertEquals(self.mw.process_spider_input(res, self.spider),
                          None)

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
                          None)
開發者ID:serkanh，項目名稱:scrapy，代碼行數:32，代碼來源:test_spidermiddleware_httperror.py

示例6: TestHttpErrorMiddlewareSettings

# 需要導入模塊: from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware [as 別名]
# 或者: from scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware import process_spider_input [as 別名]
class TestHttpErrorMiddlewareSettings(TestCase):
    """Similar test, but with settings"""

    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)}))
        self.req = Request('http://scrapytest.org')

        self.res200 = Response('http://scrapytest.org', status=200)
        self.res200.request = self.req
        self.res404 = Response('http://scrapytest.org', status=404)
        self.res404.request = self.req
        self.res402 = Response('http://scrapytest.org', status=402)
        self.res402.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(None,
                self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError,
                self.mw.process_spider_input, self.res404, self.spider)
        self.assertEquals(None,
                self.mw.process_spider_input(self.res402, self.spider))

    def test_meta_overrides_settings(self):
        request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEquals(None,
            self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError,
                self.mw.process_spider_input, res402, self.spider)

    def test_spider_override_settings(self):
        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None,
            self.mw.process_spider_input(self.res404, self.spider))
        self.assertRaises(HttpError,
                self.mw.process_spider_input, self.res402, self.spider)
開發者ID:dvska，項目名稱:scrapy，代碼行數:44，代碼來源:test_spidermiddleware_httperror.py


注：本文中的scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware.process_spider_input方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。