Python Request.set_proxy方法代码示例

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
    def test_http_doubleslash(self):
        # Checks the presence of any unnecessary double slash in url does not
        # break anything. Previously, a double slash directly after the host
        # could could cause incorrect parsing.
        h = urllib.request.AbstractHTTPHandler()
        o = h.parent = MockOpener()

        data = ""
        ds_urls = [
            "http://example.com/foo/bar/baz.html",
            "http://example.com//foo/bar/baz.html",
            "http://example.com/foo//bar/baz.html",
            "http://example.com/foo/bar//baz.html"
            ]

        for ds_url in ds_urls:
            ds_req = Request(ds_url, data)

            # Check whether host is determined correctly if there is no proxy
            np_ds_req = h.do_request_(ds_req)
            self.assertEqual(np_ds_req.unredirected_hdrs["Host"],"example.com")

            # Check whether host is determined correctly if there is a proxy
            ds_req.set_proxy("someproxy:3128",None)
            p_ds_req = h.do_request_(ds_req)
            self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com")

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
    def urlopen(self, url, param=None, data=None, headers={}, proxies={}, timeout=None, encoding='utf8', errors='strict'):
        """ 打开目标链接, 返回一个 HttpResponse对象.

        @url(str/Request): 目标链接.
        @param(str/dict/pairs tuple): query string.
        @data(bytes/str/dict): post data.
        @headers(dict): http request headers.
        @proxies(dict): 代理, 如:{'http': 'xx.xx.xx.xx:3128', 'https': 'xxx.xxx.xxx.xxx:8080'}.
        @timeout(int): http request timeout.
        @encoding/errors(str): url编码.
        """
        if param:
            full_url = isinstance(url, Request) and url.get_full_url() or url
            url_parse_dict = urlparse(full_url)._asdict()
            query_param = url_parse_dict.get('query') + (isinstance(param, str) and param or urlencode(param, encoding, errors))
            url_parse_dict['query'] = query_param
            full_url = urlunparse(url_parse_dict.values())
            request = Request(full_url)
        else:
            request = isinstance(url, Request) and url or Request(url)
        if data:
            if isinstance(data, bytes):
                request.data = data
            elif isinstance(data, str):
                request.data = data.encode(encoding, errors)
            else:
                request.data = urlencode(data).encode(encoding, errors)
        for key, value in headers.items():
            request.add_header(key, value)
        for proxy_type, proxy_host in proxies.items():
            request.set_proxy(proxy_host, proxy_type)
        self.last_request = request
        self.last_response = self.opener.open(request, timeout=timeout)
        return self.last_response

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
    def _update_apart(self, url):
        """ Обновление цены  """

        print('Update price ',url)
        req_url = Request(url=url)
        req_url.add_header(
                'User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:45.0) Gecko/20100101 Firefox/45.0')


        while (self.len_proxy_list >= self.proxy_number):

            self.proxy_ip = self.proxy_list[self.proxy_number]
            print(self.proxy_ip)

            req_url.set_proxy(self.proxy_ip, 'http')
            cond, page = self._open_url(req_url)
            if cond:
                apart = Apartment.objects.get(link=url)
                params = self._parsing_page_ad(page)
                apart_price_history = HistoryApartmentPrice.objects.create(apartment=apart, price=apart.price)
                apart.price = params['price']
                apart.save()
                return True
            else:
                self.proxy_number += 1
                continue

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
 def urlopen(*a, **kw):
     proxies = kw.pop('proxies', None)
     if proxies is None:
         return _urlopen(*a, **kw)
     r = Request(a[0])
     for k, v in proxies.items():
         r.set_proxy(v, k)
     return _urlopen(r, *a[1:], **kw)

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
class RequestTests(unittest.TestCase):

    def setUp(self):
        self.get = Request("http://www.python.org/~jeremy/")
        self.post = Request("http://www.python.org/~jeremy/",
                            "data",
                            headers={"X-Test": "test"})

    def test_method(self):
        self.assertEqual("POST", self.post.get_method())
        self.assertEqual("GET", self.get.get_method())

    def test_add_data(self):
        self.assertFalse(self.get.has_data())
        self.assertEqual("GET", self.get.get_method())
        self.get.add_data("spam")
        self.assertTrue(self.get.has_data())
        self.assertEqual("POST", self.get.get_method())

    def test_get_full_url(self):
        self.assertEqual("http://www.python.org/~jeremy/",
                         self.get.get_full_url())

    def test_selector(self):
        self.assertEqual("/~jeremy/", self.get.get_selector())
        req = Request("http://www.python.org/")
        self.assertEqual("/", req.get_selector())

    def test_get_type(self):
        self.assertEqual("http", self.get.get_type())

    def test_get_host(self):
        self.assertEqual("www.python.org", self.get.get_host())

    def test_get_host_unquote(self):
        req = Request("http://www.%70ython.org/")
        self.assertEqual("www.python.org", req.get_host())

    def test_proxy(self):
        self.assertFalse(self.get.has_proxy())
        self.get.set_proxy("www.perl.org", "http")
        self.assertTrue(self.get.has_proxy())
        self.assertEqual("www.python.org", self.get.get_origin_req_host())
        self.assertEqual("www.perl.org", self.get.get_host())

    def test_wrapped_url(self):
        req = Request("<URL:http://www.python.org>")
        self.assertEqual("www.python.org", req.get_host())

    def test_urlwith_fragment(self):
        req = Request("http://www.python.org/?qs=query#fragment=true")
        self.assertEqual("/?qs=query", req.get_selector())
        req = Request("http://www.python.org/#fun=true")
        self.assertEqual("/", req.get_selector())

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
    def _add_apartment(self, url):
        """ Добавление обьявления о квартире в базу """

        print(url)
        req_url = Request(url=url)
        req_url.add_header(
                'User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:45.0) Gecko/20100101 Firefox/45.0')


        while (self.len_proxy_list >= self.proxy_number):

            self.proxy_ip = self.proxy_list[self.proxy_number]
            print(self.proxy_ip)

            req_url.set_proxy(self.proxy_ip, 'http')
            cond, page = self._open_url(req_url)
            if cond:
                params = self._parsing_page_ad(page)
                self._save_apartment(params, url)
                return True
            else:
                self.proxy_number += 1
                continue

# 需要导入模块: from urllib.request import Request [as 别名]
# 或者: from urllib.request.Request import set_proxy [as 别名]
 def __request(self, **kargs):
     request = Request(self.__url(**kargs),
                       headers={'User-Agent': 'Uykfe ([email protected])'})
     if self.__proxy:
         request.set_proxy(self.__proxy, 'http')
     return request