Python http.Response类代码示例

 def test_request_cacheability(self):
     res0 = Response(self.request.url, status=200,
                     headers={'Expires': self.tomorrow})
     req0 = Request('http://example.com')
     req1 = req0.replace(headers={'Cache-Control': 'no-store'})
     req2 = req0.replace(headers={'Cache-Control': 'no-cache'})
     with self._middleware() as mw:
         # response for a request with no-store must not be cached
         res1 = self._process_requestresponse(mw, req1, res0)
         self.assertEqualResponse(res1, res0)
         assert mw.storage.retrieve_response(self.spider, req1) is None
         # Re-do request without no-store and expect it to be cached
         res2 = self._process_requestresponse(mw, req0, res0)
         assert 'cached' not in res2.flags
         res3 = mw.process_request(req0, self.spider)
         assert 'cached' in res3.flags
         self.assertEqualResponse(res2, res3)
         # request with no-cache directive must not return cached response
         # but it allows new response to be stored
         res0b = res0.replace(body=b'foo')
         res4 = self._process_requestresponse(mw, req2, res0b)
         self.assertEqualResponse(res4, res0b)
         assert 'cached' not in res4.flags
         res5 = self._process_requestresponse(mw, req0, None)
         self.assertEqualResponse(res5, res0b)
         assert 'cached' in res5.flags

def _responses(request, status_codes):
    responses = []
    for code in status_codes:
        response = Response(request.url, status=code)
        response.request = request
        responses.append(response)
    return responses

class TestHttpErrorMiddlewareHandleAll(TestCase):
    def setUp(self):
        self.spider = BaseSpider("foo")
        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
        self.req = Request("http://scrapytest.org")

        self.res200 = Response("http://scrapytest.org", status=200)
        self.res200.request = self.req
        self.res404 = Response("http://scrapytest.org", status=404)
        self.res404.request = self.req
        self.res402 = Response("http://scrapytest.org", status=402)
        self.res402.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))

    def test_meta_overrides_settings(self):
        request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEquals(None, self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)

 def test_cached_and_stale(self):
     sampledata = [
         (200, {'Date': self.today, 'Expires': self.yesterday}),
         (200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
         (200, {'Expires': self.yesterday}),
         (200, {'Expires': self.yesterday, 'ETag': 'foo'}),
         (200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
         (200, {'Expires': self.tomorrow, 'Age': '86405'}),
         (200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}),
         # no-cache forces expiration, also revalidation if validators exists
         (200, {'Cache-Control': 'no-cache'}),
         (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}),
         (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}),
         (200, {'Cache-Control': 'no-cache,must-revalidate', 'Last-Modified': self.yesterday}),
         (200, {'Cache-Control': 'must-revalidate', 'Expires': self.yesterday, 'Last-Modified': self.yesterday}),
         (200, {'Cache-Control': 'max-age=86400,must-revalidate', 'Age': '86405'}),
     ]
     with self._middleware() as mw:
         for idx, (status, headers) in enumerate(sampledata):
             req0 = Request('http://example-%d.com' % idx)
             res0a = Response(req0.url, status=status, headers=headers)
             # cache expired response
             res1 = self._process_requestresponse(mw, req0, res0a)
             self.assertEqualResponse(res1, res0a)
             assert 'cached' not in res1.flags
             # Same request but as cached response is stale a new response must
             # be returned
             res0b = res0a.replace(body=b'bar')
             res2 = self._process_requestresponse(mw, req0, res0b)
             self.assertEqualResponse(res2, res0b)
             assert 'cached' not in res2.flags
             cc = headers.get('Cache-Control', '')
             # Previous response expired too, subsequent request to same
             # resource must revalidate and succeed on 304 if validators
             # are present
             if 'ETag' in headers or 'Last-Modified' in headers:
                 res0c = res0b.replace(status=304)
                 res3 = self._process_requestresponse(mw, req0, res0c)
                 self.assertEqualResponse(res3, res0b)
                 assert 'cached' in res3.flags
                 # get cached response on server errors unless must-revalidate
                 # in cached response
                 res0d = res0b.replace(status=500)
                 res4 = self._process_requestresponse(mw, req0, res0d)
                 if 'must-revalidate' in cc:
                     assert 'cached' not in res4.flags
                     self.assertEqualResponse(res4, res0d)
                 else:
                     assert 'cached' in res4.flags
                     self.assertEqualResponse(res4, res0b)
             # Requests with max-stale can fetch expired cached responses
             # unless cached response has must-revalidate
             req1 = req0.replace(headers={'Cache-Control': 'max-stale'})
             res5 = self._process_requestresponse(mw, req1, res0b)
             self.assertEqualResponse(res5, res0b)
             if 'no-cache' in cc or 'must-revalidate' in cc:
                 assert 'cached' not in res5.flags
             else:
                 assert 'cached' in res5.flags

def pytest_funcarg__mock_response(request):
    """
    Fake response to the scrape request -- we only fill out the fields used by
    the middleware for testing purposes
    """
    scrape_request = request.getfuncargvalue("scrape_request")
    mock_response = Response('http://test.com')
    mock_response.request = scrape_request
    return mock_response

    def setUp(self):
        self.spider = BaseSpider("foo")
        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
        self.req = Request("http://scrapytest.org")

        self.res200 = Response("http://scrapytest.org", status=200)
        self.res200.request = self.req
        self.res404 = Response("http://scrapytest.org", status=404)
        self.res404.request = self.req
        self.res402 = Response("http://scrapytest.org", status=402)
        self.res402.request = self.req

    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
        self.req = Request('http://scrapytest.org')

        self.res200 = Response('http://scrapytest.org', status=200)
        self.res200.request = self.req
        self.res404 = Response('http://scrapytest.org', status=404)
        self.res404.request = self.req
        self.res402 = Response('http://scrapytest.org', status=402)
        self.res402.request = self.req

    def test_empty_content_type(self):
        name = "ebay4"
        spider = self.smanager.create(name)
        generic_form_request = list(spider.start_requests())[0]

        response = Response(url="http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
                            body=open(join(_PATH, "data", "ebay_advanced_search.html")).read())
        response.request = generic_form_request
        # must not raise an error
        for result in spider.parse(response):
            pass

def test_hs_mware_process_spider_output_filter_request(hs_mware):
    response = Response('http://resp-url')
    # provide a response and a new request in result
    child_response = Response('http://resp-url-child')
    child_response.request = Request('http://resp-url-child-req')
    child_request = Request('http://req-url-child')
    hs_mware._seen = WeakKeyDictionary({response: 'riq'})
    result = list(hs_mware.process_spider_output(
        response, [child_response, child_request], Spider('test')))
    assert len(result) == 2
    # make sure that we update hsparent meta only for requests
    assert result[0].meta.get(HS_PARENT_ID_KEY) is None
    assert result[1].meta[HS_PARENT_ID_KEY] == 'riq'

def test_hs_middlewares(hs_downloader_middleware, hs_spider_middleware):
    assert hs_spider_middleware._seen_requests == WeakKeyDictionary()
    assert hs_downloader_middleware._seen_requests == WeakKeyDictionary()
    assert hs_spider_middleware._seen_requests is hs_downloader_middleware._seen_requests

    spider = Spider('test')
    url = 'http://resp-url'
    request_0 = Request(url)
    response_0 = Response(url)

    hs_downloader_middleware.process_request(request_0, spider)

    assert HS_REQUEST_ID_KEY not in request_0.meta
    assert HS_PARENT_ID_KEY not in request_0.meta
    assert len(hs_spider_middleware._seen_requests) == 0
    assert len(hs_downloader_middleware._seen_requests) == 0

    hs_downloader_middleware.process_response(request_0, response_0, spider)

    assert request_0.meta[HS_REQUEST_ID_KEY] == 0
    assert request_0.meta[HS_PARENT_ID_KEY] is None
    assert hs_spider_middleware._seen_requests[request_0] == 0

    response_0.request = request_0
    request_1 = Request(url)
    request_2 = Request(url)
    item1 = {}
    item2 = Item()
    output = [request_1, request_2, item1, item2]
    processed_output = list(hs_spider_middleware.process_spider_output(response_0, output, spider))

    assert processed_output[0] is request_1
    assert request_1.meta[HS_PARENT_ID_KEY] == 0
    assert processed_output[1] is request_2
    assert request_2.meta[HS_PARENT_ID_KEY] == 0
    assert processed_output[2] is item1
    assert processed_output[3] is item2

    response_1 = Response(url)
    hs_downloader_middleware.process_request(request_1, spider)
    hs_downloader_middleware.process_response(request_1, response_1, spider)
    assert request_1.meta[HS_REQUEST_ID_KEY] == 1
    assert request_1.meta[HS_PARENT_ID_KEY] == 0

    response_2 = Response(url)
    hs_downloader_middleware.process_request(request_2, spider)
    hs_downloader_middleware.process_response(request_2, response_2, spider)
    assert request_2.meta[HS_REQUEST_ID_KEY] == 2
    assert request_2.meta[HS_PARENT_ID_KEY] == 0

class TestHttpErrorMiddleware(TestCase):

    def setUp(self):
        self.spider = BaseSpider()
        self.mw = HttpErrorMiddleware()
        self.req = Request('http://scrapytest.org')

        self.res200 = Response('http://scrapytest.org', status=200)
        self.res200.request = self.req
        self.res404 = Response('http://scrapytest.org', status=404)
        self.res404.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(self.mw.process_spider_input(self.res200, self.spider),
                          None)

        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
                          [])

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})

        self.assertEquals(self.mw.process_spider_input(res, self.spider),
                          None)

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
                          None)

class TestHttpErrorMiddleware(TestCase):
    def setUp(self):
        self.spider = BaseSpider("foo")
        self.mw = HttpErrorMiddleware(Settings({}))
        self.req = Request("http://scrapytest.org")

        self.res200 = Response("http://scrapytest.org", status=200)
        self.res200.request = self.req
        self.res404 = Response("http://scrapytest.org", status=404)
        self.res404.request = self.req

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)

    def test_process_spider_exception(self):
        self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
        self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider))

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        self.assertEquals(None, self.mw.process_spider_input(res, self.spider))

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))

def test_hs_mware_process_spider_input(hs_mware):
    response = Response('http://resp-url')
    response.request = Request('http://req-url')
    hs_mware.process_spider_input(response, Spider('test'))
    assert hs_mware.pipe_writer.write_request.call_count == 1
    args = hs_mware.pipe_writer.write_request.call_args[1]
    assert args == {
        'duration': 0,
        'fp': request_fingerprint(response.request),
        'method': 'GET',
        'parent': None,
        'rs': 0,
        'status': 200,
        'url': 'http://resp-url'
    }
    assert hs_mware._seen == WeakKeyDictionary({response: 0})

 def test_response_cacheability(self):
     responses = [
         # 304 is not cacheable no matter what servers sends
         (False, 304, {}),
         (False, 304, {'Last-Modified': self.yesterday}),
         (False, 304, {'Expires': self.tomorrow}),
         (False, 304, {'Etag': 'bar'}),
         (False, 304, {'Cache-Control': 'max-age=3600'}),
         # Always obey no-store cache control
         (False, 200, {'Cache-Control': 'no-store'}),
         # invalid
         (False, 200, {'Cache-Control': 'no-store, max-age=300'}),
         # invalid
         (False, 200, {
          'Cache-Control': 'no-store', 'Expires': self.tomorrow}),
         # Ignore responses missing expiration and/or validation headers
         (False, 200, {}),
         (False, 302, {}),
         (False, 307, {}),
         (False, 404, {}),
         # Cache responses with expiration and/or validation headers
         (True, 200, {'Last-Modified': self.yesterday}),
         (True, 203, {'Last-Modified': self.yesterday}),
         (True, 300, {'Last-Modified': self.yesterday}),
         (True, 301, {'Last-Modified': self.yesterday}),
         (True, 401, {'Last-Modified': self.yesterday}),
         (True, 404, {'Cache-Control': 'public, max-age=600'}),
         (True, 302, {'Expires': self.tomorrow}),
         (True, 200, {'Etag': 'foo'}),
     ]
     with self._middleware() as mw:
         for idx, (shouldcache, status, headers) in enumerate(responses):
             req0 = Request('http://example-%d.com' % idx)
             res0 = Response(req0.url, status=status, headers=headers)
             res1 = self._process_requestresponse(mw, req0, res0)
             res304 = res0.replace(status=304)
             res2 = self._process_requestresponse(
                 mw, req0, res304 if shouldcache else res0)
             self.assertEqualResponse(res1, res0)
             self.assertEqualResponse(res2, res0)
             resc = mw.storage.retrieve_response(self.spider, req0)
             if shouldcache:
                 self.assertEqualResponse(resc, res1)
                 assert 'cached' in res2.flags and res2.status != 304
             else:
                 self.assertFalse(resc)
                 assert 'cached' not in res2.flags

    def test_parse_declaration_doc(self):
        response = Response('http://old.vtek.lt/vtek/.../deklaracija2012.doc', body='msword msword msword')
        response.request = scrapy.Request(response.url)
        response.request.meta['year'] = '2012'

        def mock_doc2xml(msword):
            assert msword == 'msword msword msword'
            return 'xml xml xml'

        with mock.patch('manoseimas.scrapy.spiders.lobbyist_declarations.doc2xml', mock_doc2xml):
            with mock.patch.object(self.spider, 'parse_declaration_xml') as p_d_x:
                list(self.spider.parse_declaration_doc(response))
                assert p_d_x.call_count == 1
                new_response = p_d_x.call_args[0][0]
                assert new_response.meta['year'] == '2012'
                assert new_response.body == 'xml xml xml'
                assert isinstance(new_response, XmlResponse)