Python util.build_grab函数代码示例

    def test_session(self):
        # Test that if Grab gets some cookies from the server
        # then it sends it back
        grab = build_grab()
        grab.setup(reuse_cookies=True)
        self.server.response['cookies'] = {'foo': 'bar'}.items()
        grab.go(self.server.get_url())
        self.assertEqual(grab.doc.cookies['foo'], 'bar')
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['Cookie'], 'foo=bar')
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['Cookie'], 'foo=bar')

        # Test reuse_cookies=False
        grab = build_grab()
        grab.setup(reuse_cookies=False)
        self.server.response['cookies'] = {'foo': 'baz'}.items()
        grab.go(self.server.get_url())
        self.assertEqual(grab.doc.cookies['foo'], 'baz')
        grab.go(self.server.get_url())
        self.assertTrue(len(self.server.request['cookies']) == 0)

        # Test something
        grab = build_grab()
        grab.setup(reuse_cookies=True)
        self.server.response['cookies'] = {'foo': 'bar'}.items()
        grab.go(self.server.get_url())
        self.assertEqual(grab.doc.cookies['foo'], 'bar')
        grab.clear_cookies()
        grab.go(self.server.get_url())
        self.assertTrue(len(self.server.request['cookies']) == 0)

    def test_unicode_post(self):
        # By default, unicode post should be converted into utf-8
        grab = build_grab()
        data = u'фыва'
        grab.setup(post=data, url=self.server.get_url())
        grab.request()
        self.assertEqual(self.server.request['data'], data.encode('utf-8'))

        # Now try cp1251 with charset option
        self.server.request['charset'] = 'cp1251'
        grab = build_grab()
        data = u'фыва'
        grab.setup(post=data, url=self.server.get_url(),
                   charset='cp1251', debug=True)
        grab.request()
        self.assertEqual(self.server.request['data'], data.encode('cp1251'))

        # Now try dict with unicode value & charset option
        self.server.request['charset'] = 'cp1251'
        grab = build_grab()
        data = u'фыва'
        grab.setup(post={'foo': data}, url=self.server.get_url(),
                   charset='cp1251', debug=True)
        grab.request()
        test = 'foo=%s' % quote(data.encode('cp1251'))
        test = test.encode('utf-8')  # py3 hack
        self.assertEqual(self.server.request['data'], test)

    def test_dash_issue(self):
        html = '<strong>&#151;</strong>'
        self.server.response['get.data'] = html
        grab = build_grab()
        grab.go(self.server.get_url())

        # By default &#[128-160]; are fixed
        self.assertFalse(grab.doc.select('//strong/text()').text()
                         == six.unichr(151))
        self.assertTrue(grab.doc.select('//strong/text()').text()
                        == six.unichr(8212))

        # disable fix-behaviour
        grab.setup(fix_special_entities=False)
        grab.go(self.server.get_url())

        # By default &#[128-160]; are fixed
        self.assertTrue(grab.doc.select('//strong/text()').text()
                        == six.unichr(151))
        self.assertFalse(grab.doc.select('//strong/text()').text()
                         == six.unichr(8212))

        # Explicitly use unicode_body func
        grab = build_grab()
        grab.go(self.server.get_url())
        #print(':::', grab.doc.unicode_body())
        self.assertTrue('&#8212;' in grab.doc.unicode_body())

    def test_load_dump(self):
        with temp_file() as tmp_file:
            grab = build_grab()
            cookies = {'foo': 'bar', 'spam': 'ham'}
            grab.setup(cookies=cookies)
            grab.go(self.server.get_url())
            grab.cookies.save_to_file(tmp_file)
            with open(tmp_file) as inp:
                self.assertEqual(set(cookies.items()),
                                 set((x['name'], x['value'])
                                     for x in json.load(inp)))

            grab = build_grab()
            cookies = {'foo': 'bar', 'spam': u'begemot'}
            grab.setup(cookies=cookies)
            grab.go(self.server.get_url())
            grab.cookies.save_to_file(tmp_file)
            with open(tmp_file) as inp:
                self.assertEqual(set(cookies.items()),
                                 set((x['name'], x['value'])
                                     for x in json.load(inp)))

            # Test load cookies
            grab = build_grab()
            cookies = [{'name': 'foo', 'value': 'bar',
                        'domain': self.server.address},
                       {'name': 'spam', 'value': u'begemot',
                        'domain': self.server.address}]
            with open(tmp_file, 'w') as out:
                json.dump(cookies, out)
            grab.cookies.load_from_file(tmp_file)
            self.assertEqual(set(grab.cookies.items()),
                             set((x['name'], x['value']) for x in cookies))

 def test_make_url_absolute(self):
     grab = build_grab()
     self.server.response['get.data'] = '<base href="http://foo/bar/">'
     grab.go(self.server.get_url())
     absolute_url = grab.make_url_absolute('/foobar', resolve_base=True)
     self.assertEqual(absolute_url, 'http://foo/foobar')
     grab = build_grab()
     absolute_url = grab.make_url_absolute('/foobar')
     self.assertEqual(absolute_url, '/foobar')

 def test_adopt(self):
     grab = build_grab()
     self.server.response['get.data'] = 'Moon'
     grab.go(self.server.get_url())
     grab2 = build_grab()
     self.assertEqual(grab2.config['url'], None)
     grab2.adopt(grab)
     self.assertTrue(b'Moon' in grab2.doc.body)
     self.assertEqual(grab2.config['url'], self.server.get_url())

    def test_empty_document(self):
        self.server.response['get.data'] = 'oops'
        grab = build_grab()
        grab.go(self.server.get_url())
        grab.xpath_exists('//anytag')

        self.server.response['get.data'] = '<frameset></frameset>'
        grab = build_grab()
        grab.go(self.server.get_url())
        grab.xpath_exists('//anytag')

    def test_useragent(self):
        grab = build_grab()

        # Null value activates default random user-agent
        # For some transports it just allow them to send default user-agent
        # like in Kit transport case
        grab = build_grab()
        grab.setup(user_agent=None)
        grab.go(self.server.get_url())
        self.assertTrue(len(self.server.request['headers']) > 0)
        self.assertFalse('PycURL' in
                         self.server.request['headers']['user-agent'])

        # By default user_agent is None => random user agent is generated
        grab = build_grab()
        grab.go(self.server.get_url())
        self.assertTrue(len(self.server.request['headers']) > 0)
        self.assertFalse('PycURL' in
                         self.server.request['headers']['user-agent'])

        # Simple case: setup user agent manually
        grab.setup(user_agent='foo')
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['user-agent'], 'foo')

        with temp_file() as ua_file:
            # user agent from file should be loaded
            with open(ua_file, 'w') as out:
                out.write('GOD')
            grab.setup(user_agent=None, user_agent_file=ua_file)
            grab.go(self.server.get_url())
            self.assertEqual(self.server.request['headers']['user-agent'],
                             'GOD')

        with temp_file() as ua_file:
            # random user agent from file should be loaded
            with open(ua_file, 'w') as out:
                out.write('GOD1\nGOD2')
            grab.setup(user_agent=None, user_agent_file=ua_file)
            grab.go(self.server.get_url())
            self.assertTrue(self.server.request['headers']['user-agent']
                            in ('GOD1', 'GOD2'))
            agent = grab.config['user_agent']

        # User-agent should not change
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['user-agent'], agent)

        # User-agent should not change
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['user-agent'], agent)

    def test_cdata_issue(self):
        self.server.response['data'] = XML

        # By default HTML DOM builder is used
        # It handles CDATA incorrectly
        grab = build_grab()
        grab.go(self.server.get_url())
        self.assertEqual(None, grab.xpath_one('//weight').text)
        self.assertEqual(None, grab.doc.tree.xpath('//weight')[0].text)

        # But XML DOM builder produces valid result
        # self.assertEqual(None, grab.xpath_one('//weight').text)
        grab = build_grab(content_type='xml')
        grab.go(self.server.get_url())
        self.assertEqual('30', grab.doc.tree.xpath('//weight')[0].text)

    def test_useragent_simple(self):
        grab = build_grab()

        # Simple case: setup user agent manually
        grab.setup(user_agent='foo')
        grab.go(self.server.get_url())
        self.assertEqual(self.server.request['headers']['user-agent'], 'foo')

 def test_grab_parse_defensedxml(self):
     with temp_dir() as tmp_dir:
         injection_path = os.path.join(tmp_dir, 'injection')
         with open(injection_path, 'w') as out:
             out.write('Hey there!')
         # Prepare file:// URL valid for both linux and windows
         injection_url = 'file:///%s' % (injection_path.lstrip('/')
                                         .replace('\\', '/'))
         bad_xml = (
             '<!DOCTYPE external ['
             '<!ENTITY ee SYSTEM "' + injection_url + '">'
             ']>'
             '<root>&ee;</root>'
         ).encode()
         xml_file = os.path.join(tmp_dir, 'bad.xml')
         # On windows, use slashed instead of backslashes to avoid error:
         # Invalid file://hostname/, expected localhost or 127.0.0.1 or none
         if '\\' in xml_file:
             xml_file = xml_file.replace('\\', '/')
         with open(xml_file, 'wb') as out:
             out.write(bad_xml)
         grab = build_grab(content_type='xml')
         file_url = 'file://%s' % xml_file
         grab.go(file_url)
         self.assertRaises(EntitiesForbidden, grab.doc, '//title')

 def test_invalid_charset(self):
     html = '''<head><meta http-equiv="Content-Type"
                 content="text/html; charset=windows-874">'
                 </head><body>test</body>'''
     self.server.response['get.data'] = html
     grab = build_grab()
     grab.go(self.server.get_url())

    def test_assign_unicode_to_body(self):
        grab = build_grab()
        grab.doc.body = b'abc'
        grab.doc.body = b'def'

        with self.assertRaises(GrabMisuseError):
            grab.doc.body = u'Спутник'

    def test_set_methods(self):
        grab = build_grab()
        self.server.response['get.data'] = FORMS_HTML
        grab.go(self.server.get_url())

        # pylint: disable=protected-access
        self.assertEqual(grab.doc._lxml_form, None)
        # pylint: enable=protected-access

        grab.doc.set_input('gender', '1')
        # pylint: disable=no-member,protected-access
        self.assertEqual('common_form', grab.doc._lxml_form.get('id'))
        # pylint: enable=no-member,protected-access

        # pylint: disable=no-member,protected-access
        self.assertRaises(KeyError,
                          lambda: grab.doc.set_input('query', 'asdf'))
        # pylint: enable=no-member,protected-access

        grab.doc._lxml_form = None # pylint: disable=protected-access
        grab.doc.set_input_by_id('search_box', 'asdf')
        # pylint: disable=no-member,protected-access
        self.assertEqual('search_form', grab.doc._lxml_form.get('id'))
        # pylint: enable=no-member,protected-access

        grab.doc.choose_form(xpath='//form[@id="common_form"]')
        grab.doc.set_input_by_number(0, 'asdf')

        # pylint: disable=no-member,protected-access
        grab.doc._lxml_form = None
        grab.doc.set_input_by_xpath('//*[@name="gender"]', '2')
        self.assertEqual('common_form', grab.doc._lxml_form.get('id'))

    def test_submit(self):
        grab = build_grab()
        self.server.response['get.data'] = POST_FORM % self.server.get_url()
        grab.go(self.server.get_url())
        grab.doc.set_input('name', 'Alex')
        grab.submit()
        self.assert_equal_qs(self.server.request['data'],
                             b'name=Alex&secret=123')

        # Default submit control
        self.server.response['get.data'] = MULTIPLE_SUBMIT_FORM
        grab.go(self.server.get_url())
        grab.submit()
        self.assert_equal_qs(self.server.request['data'],
                             b'secret=123&submit1=submit1')

        # Selected submit control
        self.server.response['get.data'] = MULTIPLE_SUBMIT_FORM
        grab.go(self.server.get_url())
        grab.submit(submit_name='submit2')
        self.assert_equal_qs(self.server.request['data'],
                             b'secret=123&submit2=submit2')

        # Default submit control if submit control name is invalid
        self.server.response['get.data'] = MULTIPLE_SUBMIT_FORM
        grab.go(self.server.get_url())
        grab.submit(submit_name='submit3')
        self.assert_equal_qs(self.server.request['data'],
                             b'secret=123&submit1=submit1')