Python request.Response类代码示例

    def test_html_scraper_links_base_href(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "basehref.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual("utf-8", scrape_result.encoding)

        self.assertEqual(
            {
                "http://cdn.example.com/stylesheet1.css",
                "http://www.example.com/stylesheet2.css",
                "http://example.com/a/stylesheet3.css",
                "http://example.com/a/dir/image1.png",
                "http://example.com/dir/image2.png",
                "http://example.net/image3.png",
                "http://example.com/dir/image4.png",
            },
            inline_urls,
        )
        self.assertEqual({"http://example.com/a/"}, linked_urls)

    def test_adjust_extension(self):
        writer = AntiClobberFileWriter(self.get_path_namer(), adjust_extension=True)

        test_data = [
            ('text/html', '/mordor', 'mordor.html'),
            ('text/html', '/mordor?ring.asp', 'mordor?ring.asp.html'),
            ('text/html', '/mordor?ring.htm', 'mordor?ring.htm'),
            ('text/plain', '/static/my_file.txt', 'static/my_file.txt'),
            ('text/css', '/static/style.css', 'static/style.css'),
            ('text/css', '/static/style.css?hamster.exe', 'static/style.css?hamster.exe.css'),
            ('text/html', '/static/mojibake.html', 'static/mojibake.html'),
            ('text/html', '/static/mojibake.html?dolphin.png', 'static/mojibake.html?dolphin.png.html'),
        ]

        for mime_type, path, filename in test_data:
            session = writer.session()

            request = HTTPRequest('http://example.com' + path)
            response = HTTPResponse(status_code=200, reason='OK', request=request)
            response.fields['Content-Type'] = mime_type

            session.process_request(request)
            session.process_response(response)
            session.save_document(response)

            print(filename, list(os.walk('.')))
            self.assertTrue(os.path.exists(filename))

    def test_javascript_heavy_inline_monstrosity(self):
        scraper = JavaScriptScraper()
        request = Request('http://example.com/test.js')
        response = Response(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH,
                                          'testing', 'samples',
                                          'twitchplayspokemonfirered.html')
            with open(html_file_path, 'rb') as in_file:
                in_file.seek(0x147)
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertIn(
            'http://cdn.bulbagarden.net/upload/archive/a/a4/'
            '20090718115357%21195Quagsire.png',
            inline_urls
        )
        self.assertIn(
            'http://www.google.com/url?q=http%3A%2F%2Fwww.reddit.com%2F'
            'user%2FGoldenSandslash15&sa=D&sntz=1&'
            'usg=AFQjCNElFBxZYdNm5mWoRSncf5tbdIJQ-A',
            linked_urls
        )

        print('\n'.join(inline_urls))
        print('\n'.join(linked_urls))

    def test_html_wrong_charset(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "kcna.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual("utf-16-le", scrape_result.encoding)

        self.assertEqual(
            {
                "http://example.com/utm/__utm.js",
                "http://example.com/Knewskage.gif",
                "http://example.com/Lline.gif",
                "http://example.com/Sline.gif",
                "http://example.com/korean01.gif",
                "http://example.com/korean02.gif",
                "http://example.com/english01.gif",
                "http://example.com/english02.gif",
                "http://example.com/Tongsinkage.gif",
                "http://example.com/Knewskage.gif",
            },
            inline_urls,
        )
        self.assertEqual({"http://example.com/index-k.htm", "http://example.com/index-e.htm"}, linked_urls)

    def test_sitemap_scraper_xml_index(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/sitemap.xml")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(
                b"""<?xml version="1.0" encoding="UTF-8"?>
                <sitemapindex
                xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                   <sitemap>
                      <loc>http://www.example.com/sitemap1.xml.gz</loc>
                      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
                   </sitemap>
                </sitemapindex>
            """
            )

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://www.example.com/sitemap1.xml.gz"}, linked_urls)
        self.assertFalse(inline_urls)

    def test_sitemap_scraper_xml(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/sitemap.xml")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(
                b"""<?xml version="1.0" encoding="UTF-8"?>
                <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                   <url>
                      <loc>http://www.example.com/</loc>
                      <lastmod>2005-01-01</lastmod>
                      <changefreq>monthly</changefreq>
                      <priority>0.8</priority>
                   </url>
                </urlset>
            """
            )

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://www.example.com/"}, linked_urls)
        self.assertFalse(inline_urls)

    def test_http_response(self):
        response = Response(200, 'OK', version='HTTP/1.0')
        response.fields['hello'] = 'world'

        new_response = HTTPResponseInfoWrapper(response)
        info = new_response.info()

        self.assertEqual('world', info.get('hello'))

    def test_response(self):
        response = Response(200, 'OK')
        response.fields['Cake'] = 'dolphin'

        self.assertEqual(
            (b'HTTP/1.1 200 OK\r\n'
             b'Cake: dolphin\r\n'
             b'\r\n'),
            response.to_bytes()
        )

    def test_js_detect(self):
        self.assertTrue(JavaScriptReader.is_file(io.BytesIO("var a = 1;".encode("utf-16le"))))
        self.assertTrue(JavaScriptReader.is_file(io.BytesIO("setTimeout(".encode("utf-16le"))))
        self.assertFalse(JavaScriptReader.is_file(io.BytesIO("hello world!".encode("utf-16le"))))
        self.assertFalse(JavaScriptReader.is_file(io.BytesIO(b"<html><body>hello")))
        self.assertTrue(JavaScriptReader.is_file(io.BytesIO(b"<html><body>hello")) is VeryFalse)

        response = Response(200, "OK")
        response.fields["Content-Type"] = "application/javascript"
        self.assertTrue(JavaScriptReader.is_response(response))

        response = Response(200, "OK")
        response.fields["Content-Type"] = "image/png"
        self.assertFalse(JavaScriptReader.is_response(response))

    def test_html_detect(self):
        self.assertTrue(HTMLReader.is_file(
            io.BytesIO('<html><body>hi</body></html>'.encode('utf-16le'))
        ))
        self.assertFalse(HTMLReader.is_file(
            io.BytesIO('hello world!'.encode('utf-16le'))
        ))
        self.assertTrue(HTMLReader.is_file(
            io.BytesIO(b'<title>hello</title>hi')
        ))
        self.assertTrue(HTMLReader.is_file(
            io.BytesIO(b'<html><body>hello')
        ))
        self.assertTrue(HTMLReader.is_file(
            io.BytesIO(
                b'The document has moved <a href="somewhere.html">here</a>'
            )
        ))
        self.assertTrue(
            HTMLReader.is_url(URLInfo.parse('example.com/index.htm'))
        )
        self.assertTrue(
            HTMLReader.is_url(URLInfo.parse('example.com/index.html'))
        )
        self.assertTrue(
            HTMLReader.is_url(URLInfo.parse('example.com/index.dhtm'))
        )
        self.assertTrue(
            HTMLReader.is_url(URLInfo.parse('example.com/index.xhtml'))
        )
        self.assertTrue(
            HTMLReader.is_url(URLInfo.parse('example.com/index.xht'))
        )
        self.assertFalse(
            HTMLReader.is_url(URLInfo.parse('example.com/image.jpg'))
        )
        self.assertTrue(
            HTMLReader.is_request(Request('example.com/index.html'))
        )
        self.assertFalse(
            HTMLReader.is_request(Request('example.com/image.jpg'))
        )

        response = Response(200, 'OK')
        response.fields['Content-Type'] = 'text/html'
        self.assertTrue(HTMLReader.is_response(response))

        response = Response(200, 'OK')
        response.fields['Content-Type'] = 'image/png'
        self.assertFalse(HTMLReader.is_response(response))

    def test_html_scraper_reject_type(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "many_urls.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response, link_type=LinkType.css)
        self.assertFalse(scrape_result)

    def test_warc_max_size_and_append(self):
        file_prefix = 'asdf'

        with open('asdf-00000.warc', 'w'):
            pass

        with open('asdf-00001.warc', 'w'):
            pass

        warc_recorder = WARCRecorder(
            file_prefix,
            params=WARCRecorderParams(
                compress=False,
                max_size=1,
                appending=True
            ),
        )

        request = HTTPRequest('http://example.com/1')
        request.address = ('0.0.0.0', 80)
        response = HTTPResponse(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b'BLAH')

        session = warc_recorder.new_http_recorder_session()
        session.begin_request(request)
        session.request_data(request.to_bytes())
        session.end_request(request)
        session.begin_response(response)
        session.response_data(response.to_bytes())
        session.response_data(response.body.content())
        session.end_response(response)
        session.close()

        warc_recorder.close()

        self.assertTrue(os.path.exists('asdf-00000.warc'))
        self.assertTrue(os.path.exists('asdf-00001.warc'))
        self.assertTrue(os.path.exists('asdf-00002.warc'))
        self.assertTrue(os.path.exists('asdf-00003.warc'))
        self.assertTrue(os.path.exists('asdf-meta.warc'))

        self.assertEqual(0, os.path.getsize('asdf-00000.warc'))
        self.assertEqual(0, os.path.getsize('asdf-00001.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-00002.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-00003.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-meta.warc'))

    def test_css_scraper_reject_type(self):
        scraper = CSSScraper()
        request = Request('http://example.com/styles.css')
        response = Response(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH,
                                          'testing', 'samples', 'styles.css')
            with open(html_file_path, 'rb') as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response,
                                       link_type=LinkType.html)
        self.assertFalse(scrape_result)

    def test_progress_http(self):
        progress = ProgressPrinter(stream=sys.stdout)

        request = HTTPRequest('http://example.com')
        response = HTTPResponse(206, 'OK')
        response.fields['Content-Size'] = '1024'
        response.fields['Content-Range'] = 'bytes 10-/2048'

        progress.update_from_begin_request(request)
        progress.update_from_begin_response(response)

        for dummy in range(100):
            progress.update_with_data(b'abc')

        progress.update_from_end_response(response)

    def test_sitemap_scraper_robots(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/robots.txt")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b"Sitemap: http://example.com/sitemap00.xml")

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://example.com/sitemap00.xml"}, linked_urls)
        self.assertFalse(inline_urls)