Python Response.body方法代码示例

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_to_dict_body(self):
        request = Request()
        request.body = Body()
        request_dict = request.to_dict()

        self.assertTrue(request_dict['body'])
        request.body.close()

        request = Request()
        request.body = NotImplemented
        request_dict = request.to_dict()

        self.assertFalse(request_dict['body'])

        response = Response()
        response.body = Body()
        response_dict = response.to_dict()

        self.assertTrue(response_dict['body'])
        response.body.close()

        response = Response()
        response.body = NotImplemented
        response_dict = response.to_dict()

        self.assertFalse(response_dict['body'])

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_wrong_charset(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "kcna.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual("utf-16-le", scrape_result.encoding)

        self.assertEqual(
            {
                "http://example.com/utm/__utm.js",
                "http://example.com/Knewskage.gif",
                "http://example.com/Lline.gif",
                "http://example.com/Sline.gif",
                "http://example.com/korean01.gif",
                "http://example.com/korean02.gif",
                "http://example.com/english01.gif",
                "http://example.com/english02.gif",
                "http://example.com/Tongsinkage.gif",
                "http://example.com/Knewskage.gif",
            },
            inline_urls,
        )
        self.assertEqual({"http://example.com/index-k.htm", "http://example.com/index-e.htm"}, linked_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_sitemap_scraper_xml_index(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/sitemap.xml")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(
                b"""<?xml version="1.0" encoding="UTF-8"?>
                <sitemapindex
                xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                   <sitemap>
                      <loc>http://www.example.com/sitemap1.xml.gz</loc>
                      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
                   </sitemap>
                </sitemapindex>
            """
            )

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://www.example.com/sitemap1.xml.gz"}, linked_urls)
        self.assertFalse(inline_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_scraper_links_base_href(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "basehref.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual("utf-8", scrape_result.encoding)

        self.assertEqual(
            {
                "http://cdn.example.com/stylesheet1.css",
                "http://www.example.com/stylesheet2.css",
                "http://example.com/a/stylesheet3.css",
                "http://example.com/a/dir/image1.png",
                "http://example.com/dir/image2.png",
                "http://example.net/image3.png",
                "http://example.com/dir/image4.png",
            },
            inline_urls,
        )
        self.assertEqual({"http://example.com/a/"}, linked_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_javascript_heavy_inline_monstrosity(self):
        scraper = JavaScriptScraper()
        request = Request('http://example.com/test.js')
        response = Response(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH,
                                          'testing', 'samples',
                                          'twitchplayspokemonfirered.html')
            with open(html_file_path, 'rb') as in_file:
                in_file.seek(0x147)
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertIn(
            'http://cdn.bulbagarden.net/upload/archive/a/a4/'
            '20090718115357%21195Quagsire.png',
            inline_urls
        )
        self.assertIn(
            'http://www.google.com/url?q=http%3A%2F%2Fwww.reddit.com%2F'
            'user%2FGoldenSandslash15&sa=D&sntz=1&'
            'usg=AFQjCNElFBxZYdNm5mWoRSncf5tbdIJQ-A',
            linked_urls
        )

        print('\n'.join(inline_urls))
        print('\n'.join(linked_urls))

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_sitemap_scraper_xml(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/sitemap.xml")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(
                b"""<?xml version="1.0" encoding="UTF-8"?>
                <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                   <url>
                      <loc>http://www.example.com/</loc>
                      <lastmod>2005-01-01</lastmod>
                      <changefreq>monthly</changefreq>
                      <priority>0.8</priority>
                   </url>
                </urlset>
            """
            )

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://www.example.com/"}, linked_urls)
        self.assertFalse(inline_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_scraper_reject_type(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "many_urls.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response, link_type=LinkType.css)
        self.assertFalse(scrape_result)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_warc_max_size_and_append(self):
        file_prefix = 'asdf'

        with open('asdf-00000.warc', 'w'):
            pass

        with open('asdf-00001.warc', 'w'):
            pass

        warc_recorder = WARCRecorder(
            file_prefix,
            params=WARCRecorderParams(
                compress=False,
                max_size=1,
                appending=True
            ),
        )

        request = HTTPRequest('http://example.com/1')
        request.address = ('0.0.0.0', 80)
        response = HTTPResponse(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b'BLAH')

        session = warc_recorder.new_http_recorder_session()
        session.begin_request(request)
        session.request_data(request.to_bytes())
        session.end_request(request)
        session.begin_response(response)
        session.response_data(response.to_bytes())
        session.response_data(response.body.content())
        session.end_response(response)
        session.close()

        warc_recorder.close()

        self.assertTrue(os.path.exists('asdf-00000.warc'))
        self.assertTrue(os.path.exists('asdf-00001.warc'))
        self.assertTrue(os.path.exists('asdf-00002.warc'))
        self.assertTrue(os.path.exists('asdf-00003.warc'))
        self.assertTrue(os.path.exists('asdf-meta.warc'))

        self.assertEqual(0, os.path.getsize('asdf-00000.warc'))
        self.assertEqual(0, os.path.getsize('asdf-00001.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-00002.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-00003.warc'))
        self.assertNotEqual(0, os.path.getsize('asdf-meta.warc'))

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_sitemap_scraper_robots(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/robots.txt")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b"Sitemap: http://example.com/sitemap00.xml")

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertEqual({"http://example.com/sitemap00.xml"}, linked_urls)
        self.assertFalse(inline_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_sitemap_scraper_invalid_robots(self):
        scraper = SitemapScraper(self.get_html_parser())
        request = Request("http://example.com/robots.txt")
        response = Response(200, "OK")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b"dsfju3wrji kjasSItemapsdmjfkl wekie;er :Ads fkj3m /Dk")

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertFalse(linked_urls)
        self.assertFalse(inline_urls)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_css_scraper_reject_type(self):
        scraper = CSSScraper()
        request = Request('http://example.com/styles.css')
        response = Response(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH,
                                          'testing', 'samples', 'styles.css')
            with open(html_file_path, 'rb') as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response,
                                       link_type=LinkType.html)
        self.assertFalse(scrape_result)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_warc_recorder_rollback(self):
        warc_filename = 'asdf.warc'
        warc_prefix = 'asdf'

        with open(warc_filename, 'wb') as warc_file:
            warc_file.write(b'a' * 10)

        warc_recorder = WARCRecorder(
            warc_prefix,
            params=WARCRecorderParams(
                compress=False,
            )
        )

        request = HTTPRequest('http://example.com/')
        request.address = ('0.0.0.0', 80)
        response = HTTPResponse(200, 'OK')
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            response.body.write(b'KITTEH DOGE')

        session = warc_recorder.new_http_recorder_session()
        session.begin_request(request)
        session.request_data(request.to_bytes())

        class BadRecord(WARCRecord):
            def __init__(self, original_record):
                super().__init__()
                self.block_file = original_record.block_file
                self.fields = original_record.fields

            def __iter__(self):
                for dummy in range(1000):
                    yield b"where's my elephant?"
                raise OSError('Oops')

        session._request_record = BadRecord(session._request_record)
        original_offset = os.path.getsize(warc_filename)

        with self.assertRaises((OSError, IOError)):
            session.end_request(request)

        new_offset = os.path.getsize(warc_filename)
        self.assertEqual(new_offset, original_offset)
        self.assertFalse(os.path.exists(warc_filename + '-wpullinc'))

        _logger.debug('original offset {0}'.format(original_offset))

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_encoding_lxml_name_mismatch(self):
        """It should accept encoding names with underscore."""
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "")
        response.body = Body()
        response.fields["content-type"] = "text/html; charset=EUC_KR"

        with wpull.util.reset_file_offset(response.body):
            response.body.write("힖".encode("euc_kr"))

        scrape_info = scraper.scrape(request, response)

        self.assertTrue(scrape_info)
        self.assertEqual("euc_kr", scrape_info["encoding"])

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_serious_bad_encoding(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker, encoding_override="utf8")
        request = Request("http://example.com/")
        response = Response(200, "")
        response.body = Body()
        response.fields["content-type"] = "text/html; charset=utf8"

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "xkcd_1_evil.html")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_info = scraper.scrape(request, response)

        self.assertTrue(scrape_info)

# 需要导入模块: from wpull.protocol.http.request import Response [as 别名]
# 或者: from wpull.protocol.http.request.Response import body [as 别名]
    def test_html_not_quite_charset(self):
        element_walker = ElementWalker(css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper())
        scraper = HTMLScraper(self.get_html_parser(), element_walker)
        request = Request("http://example.com/")
        response = Response(200, "")
        response.body = Body()

        with wpull.util.reset_file_offset(response.body):
            html_file_path = os.path.join(ROOT_PATH, "testing", "samples", "videogame_top.htm")
            with open(html_file_path, "rb") as in_file:
                shutil.copyfileobj(in_file, response.body)

        scrape_result = scraper.scrape(request, response)
        inline_urls = scrape_result.inline_links
        linked_urls = scrape_result.linked_links

        self.assertIn("http://example.com/copyright_2001_2006_rtype.gif", inline_urls)
        self.assertIn("http://www.geocities.jp/gamehouse_grindcrusher/", linked_urls)