Python util.build_grab函数代码示例

    def test_session(self):
        # Test that if Grab gets some cookies from the server
        # then it sends it back
        g = build_grab()
        g.setup(reuse_cookies=True)
        self.server.response["cookies"] = {"foo": "bar"}.items()
        g.go(self.server.get_url())
        self.assertEqual(g.response.cookies["foo"], "bar")
        g.go(self.server.get_url())
        self.assertEqual(self.server.request["headers"]["Cookie"], "foo=bar")
        g.go(self.server.get_url())
        self.assertEqual(self.server.request["headers"]["Cookie"], "foo=bar")

        # Test reuse_cookies=False
        g = build_grab()
        g.setup(reuse_cookies=False)
        self.server.response["cookies"] = {"foo": "baz"}.items()
        g.go(self.server.get_url())
        self.assertEqual(g.response.cookies["foo"], "baz")
        g.go(self.server.get_url())
        self.assertTrue(len(self.server.request["cookies"]) == 0)

        # Test something
        g = build_grab()
        g.setup(reuse_cookies=True)
        self.server.response["cookies"] = {"foo": "bar"}.items()
        g.go(self.server.get_url())
        self.assertEqual(g.response.cookies["foo"], "bar")
        g.clear_cookies()
        g.go(self.server.get_url())
        self.assertTrue(len(self.server.request["cookies"]) == 0)

    def test_load_proxylist(self):
        content = '%s\n%s\n%s' % (PROXY1, PROXY2, PROXY3)
        open(TMP_FILE, 'w').write(content)

        # By default auto_change is True
        g = build_grab()
        g.load_proxylist(TMP_FILE, 'text_file')
        self.assertEqual(g.config['proxy_auto_change'], True)
        servers = set()
        for x in six.moves.range(10):
            g.go('http://yandex.ru')
            servers.add(g.config['proxy'])

        self.assertTrue(len(servers) > 1)

        # Disable auto_change
        # By default auto_init is True
        g = build_grab()
        g.load_proxylist(TMP_FILE, 'text_file', auto_change=False)
        self.assertEqual(g.config['proxy_auto_change'], False)
        servers = set()
        for x in six.moves.range(10):
            g.go('http://yandex.ru')
            servers.add(g.config['proxy'])
        self.assertEqual(len(servers), 1)

        # Disable auto_change
        # Disable auto_init
        # Proxylist will not be used by default
        g = build_grab()
        g.load_proxylist(TMP_FILE, 'text_file', auto_change=False,
                         auto_init=False)
        self.assertEqual(g.config['proxy_auto_change'], False)
        g.go(self.server.get_url())
        self.assertEqual(g.config['proxy'], None)

    def test_load_dump(self):
        g = build_grab()
        cookies = {'foo': 'bar', 'spam': 'ham'}
        g.setup(cookies=cookies)
        g.go(self.server.get_url())
        g.dump_cookies(TMP_FILE)
        self.assertEqual(set(cookies.items()),
                         set((x['name'], x['value'])
                             for x in json.load(open(TMP_FILE))))

        # Test non-ascii
        g = build_grab()
        cookies = {'foo': 'bar', 'spam': u'бегемот'}
        g.setup(cookies=cookies)
        g.go(self.server.get_url())
        g.dump_cookies(TMP_FILE)
        self.assertEqual(set(cookies.items()),
                         set((x['name'], x['value'])
                             for x in json.load(open(TMP_FILE))))

        # Test load cookies
        g = build_grab()
        cookies = [{'name': 'foo', 'value': 'bar'},
                   {'name': 'spam', 'value': u'бегемот'}]
        json.dump(cookies, open(TMP_FILE, 'w'))
        g.load_cookies(TMP_FILE)
        self.assertEqual(set(g.cookies.items()),
                         set((x['name'], x['value']) for x in cookies))

    def test_load_dump(self):
        with temp_file() as tmp_file:
            g = build_grab()
            cookies = {"foo": "bar", "spam": "ham"}
            g.setup(cookies=cookies)
            g.go(self.server.get_url())
            g.dump_cookies(tmp_file)
            self.assertEqual(set(cookies.items()), set((x["name"], x["value"]) for x in json.load(open(tmp_file))))

            g = build_grab()
            cookies = {"foo": "bar", "spam": u"begemot"}
            g.setup(cookies=cookies)
            g.go(self.server.get_url())
            g.dump_cookies(tmp_file)
            self.assertEqual(set(cookies.items()), set((x["name"], x["value"]) for x in json.load(open(tmp_file))))

            # Test load cookies
            g = build_grab()
            cookies = [
                {"name": "foo", "value": "bar", "domain": self.server.address},
                {"name": "spam", "value": u"begemot", "domain": self.server.address},
            ]
            json.dump(cookies, open(tmp_file, "w"))
            g.load_cookies(tmp_file)
            self.assertEqual(set(g.cookies.items()), set((x["name"], x["value"]) for x in cookies))

    def test_body_inmemory_false(self):
        g = build_grab()
        g.setup(body_inmemory=False)
        self.assertRaises(GrabMisuseError, lambda: g.go(self.server.get_url()))

        self.server.response['get.data'] = b'foo'
        g = build_grab()
        g.setup(body_inmemory=False)
        g.setup(body_storage_dir=TMP_DIR)
        g.go(self.server.get_url())
        self.assertTrue(os.path.exists(g.response.body_path))
        self.assertTrue(TMP_DIR in g.response.body_path)
        self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
        self.assertEqual(g.response._bytes_body, None)
        old_path = g.response.body_path

        g.go(self.server.get_url())
        self.assertTrue(old_path != g.response.body_path)

        self.server.response['get.data'] = 'foo'
        g = build_grab()
        g.setup(body_inmemory=False)
        g.setup(body_storage_dir=TMP_DIR)
        g.setup(body_storage_filename='music.mp3')
        g.go(self.server.get_url())
        self.assertTrue(os.path.exists(g.response.body_path))
        self.assertTrue(TMP_DIR in g.response.body_path)
        self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
        self.assertEqual(os.path.join(TMP_DIR, 'music.mp3'),
                         g.response.body_path)
        self.assertEqual(g.response.body, b'foo')
        self.assertEqual(g.response._bytes_body, None)

    def test_session(self):
        # Test that if Grab gets some cookies from the server
        # then it sends it back
        g = build_grab()
        g.setup(reuse_cookies=True)
        SERVER.RESPONSE['cookies'] = {'foo': 'bar'}
        g.go(SERVER.BASE_URL)
        self.assertEqual(g.response.cookies['foo'], 'bar')
        g.go(SERVER.BASE_URL)
        self.assertEqual(SERVER.REQUEST['headers']['Cookie'], 'foo=bar')
        g.go(SERVER.BASE_URL)
        self.assertEqual(SERVER.REQUEST['headers']['Cookie'], 'foo=bar')

        # Test reuse_cookies=False
        g = build_grab()
        g.setup(reuse_cookies=False)
        SERVER.RESPONSE['cookies'] = {'foo': 'baz'}
        g.go(SERVER.BASE_URL)
        self.assertEqual(g.response.cookies['foo'], 'baz')
        g.go(SERVER.BASE_URL)
        self.assertTrue(len(SERVER.REQUEST['cookies']) == 0)

        # Test something
        g = build_grab()
        g.setup(reuse_cookies=True)
        SERVER.RESPONSE['cookies'] = {'foo': 'bar'}
        g.go(SERVER.BASE_URL)
        self.assertEqual(g.response.cookies['foo'], 'bar')
        g.clear_cookies()
        g.go(SERVER.BASE_URL)
        self.assertTrue(len(SERVER.REQUEST['cookies']) == 0)

    def test_load_proxylist(self):
        content = "%s\n%s\n%s" % (PROXY1, PROXY2, PROXY3)
        open(TMP_FILE, "w").write(content)

        # By default auto_change is True
        g = build_grab()
        g.load_proxylist(TMP_FILE, "text_file")
        self.assertEqual(g.config["proxy_auto_change"], True)
        servers = set()
        for x in xrange(10):
            g.go("http://yandex.ru")
            servers.add(g.config["proxy"])

        self.assertTrue(len(servers) > 1)

        # Disable auto_change
        # By default auto_init is True
        g = build_grab()
        g.load_proxylist(TMP_FILE, "text_file", auto_change=False)
        self.assertEqual(g.config["proxy_auto_change"], False)
        servers = set()
        for x in xrange(10):
            g.go("http://yandex.ru")
            servers.add(g.config["proxy"])
        self.assertEqual(len(servers), 1)

        # Disable auto_change
        # Disable auto_init
        # Proxylist will not be used by default
        g = build_grab()
        g.load_proxylist(TMP_FILE, "text_file", auto_change=False, auto_init=False)
        self.assertEqual(g.config["proxy_auto_change"], False)
        g.go(SERVER.BASE_URL)
        self.assertEqual(g.config["proxy"], None)

    def test_load_proxylist(self):
        with temp_file() as tmp_file:
            content = "%s\n%s\n%s" % (PROXY1, PROXY2, PROXY3)
            open(tmp_file, "w").write(content)

            # By default auto_change is True
            g = build_grab()
            g.load_proxylist(tmp_file, "text_file")
            self.assertEqual(g.config["proxy_auto_change"], True)
            servers = set()
            for x in six.moves.range(10):
                g.go("http://yandex.ru")
                servers.add(g.config["proxy"])

            self.assertTrue(len(servers) > 1)

            # Disable auto_change
            # By default auto_init is True
            g = build_grab()
            g.load_proxylist(tmp_file, "text_file", auto_change=False)
            self.assertEqual(g.config["proxy_auto_change"], False)
            servers = set()
            for x in six.moves.range(10):
                g.go("http://yandex.ru")
                servers.add(g.config["proxy"])
            self.assertEqual(len(servers), 1)

            # Disable auto_change
            # Disable auto_init
            # Proxylist will not be used by default
            g = build_grab()
            g.load_proxylist(tmp_file, "text_file", auto_change=False, auto_init=False)
            self.assertEqual(g.config["proxy_auto_change"], False)
            g.go(self.server.get_url())
            self.assertEqual(g.config["proxy"], None)

    def test_body_inmemory(self):
        g = build_grab()
        g.setup(body_inmemory=False)
        self.assertRaises(GrabMisuseError, lambda: g.go(SERVER.BASE_URL))

        SERVER.RESPONSE['get'] = 'foo'
        g = build_grab()
        g.setup(body_inmemory=False)
        g.setup(body_storage_dir=TMP_DIR)
        g.go(SERVER.BASE_URL)
        self.assertTrue(os.path.exists(g.response.body_path))
        self.assertTrue(TMP_DIR in g.response.body_path)
        self.assertEqual('foo', open(g.response.body_path).read())
        old_path = g.response.body_path

        g.go(SERVER.BASE_URL)
        self.assertTrue(old_path != g.response.body_path)

        SERVER.RESPONSE['get'] = 'foo'
        g = build_grab()
        g.setup(body_inmemory=False)
        g.setup(body_storage_dir=TMP_DIR)
        g.setup(body_storage_filename='musik.mp3')
        g.go(SERVER.BASE_URL)
        self.assertTrue(os.path.exists(g.response.body_path))
        self.assertTrue(TMP_DIR in g.response.body_path)
        self.assertEqual('foo', open(g.response.body_path).read())
        self.assertEqual(os.path.join(TMP_DIR, 'musik.mp3'), g.response.body_path)

    def test_unicode_post(self):
        # By default, unicode post should be converted into utf-8
        g = build_grab()
        data = u'фыва'
        g.setup(post=data, url=self.server.get_url())
        g.request()
        self.assertEqual(self.server.request['data'], data.encode('utf-8'))

        # Now try cp1251 with charset option
        self.server.request['charset'] = 'cp1251'
        g = build_grab()
        data = u'фыва'
        g.setup(post=data, url=self.server.get_url(),
                charset='cp1251', debug=True)
        g.request()
        self.assertEqual(self.server.request['data'], data.encode('cp1251'))

        # Now try dict with unicode value & charset option
        self.server.request['charset'] = 'cp1251'
        g = build_grab()
        data = u'фыва'
        g.setup(post={'foo': data}, url=self.server.get_url(),
                charset='cp1251', debug=True)
        g.request()
        test = 'foo=%s' % quote(data.encode('cp1251'))
        test = test.encode('utf-8')  # py3 hack
        self.assertEqual(self.server.request['data'], test)

 def test_adopt(self):
     g = build_grab()
     self.server.response["get.data"] = "Moon"
     g.go(self.server.get_url())
     g2 = build_grab()
     self.assertEqual(g2.config["url"], None)
     g2.adopt(g)
     self.assertTrue(b"Moon" in g2.response.body)
     self.assertEqual(g2.config["url"], self.server.get_url())

 def test_clone(self):
     g = build_grab()
     SERVER.RESPONSE['get'] = 'Moon'
     g.go(SERVER.BASE_URL)
     self.assertTrue('Moon' in g.response.body)
     g2 = build_grab()
     self.assertEqual(g2.response, None)
     g2 = g.clone()
     self.assertTrue('Moon' in g.response.body)

 def test_make_url_absolute(self):
     g = build_grab()
     self.server.response['get.data'] = '<base href="http://foo/bar/">'
     g.go(self.server.get_url())
     absolute_url = g.make_url_absolute('/foobar', resolve_base=True)
     self.assertEqual(absolute_url, 'http://foo/foobar')
     g = build_grab()
     absolute_url = g.make_url_absolute('/foobar')
     self.assertEqual(absolute_url, '/foobar')

 def test_adopt(self):
     g = build_grab()
     SERVER.RESPONSE['get'] = 'Moon'
     g.go(SERVER.BASE_URL)
     g2 = build_grab()
     self.assertEqual(g2.config['url'], None)
     g2.adopt(g)
     self.assertTrue('Moon' in g2.response.body)
     self.assertEqual(g2.config['url'], SERVER.BASE_URL)

    def test_body_inmemory(self):
        g = build_grab()
        g.setup(body_inmemory=False)
        self.assertRaises(GrabMisuseError, lambda: g.go(SERVER.BASE_URL))

        SERVER.RESPONSE['get'] = 'foo'
        g = build_grab()
        g.setup(body_inmemory=False)
        g.setup(body_storage_dir=TMP_DIR)
        g.go(SERVER.BASE_URL)