Python OpenerDirector.open方法代码示例

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
 def test_proxy_no_proxy(self):
     os.environ['no_proxy'] = 'python.org'
     o = OpenerDirector()
     ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
     o.add_handler(ph)
     req = Request("http://www.perl.org/")
     self.assertEqual(req.get_host(), "www.perl.org")
     r = o.open(req)
     self.assertEqual(req.get_host(), "proxy.example.com")
     req = Request("http://www.python.org")
     self.assertEqual(req.get_host(), "www.python.org")
     r = o.open(req)
     self.assertEqual(req.get_host(), "www.python.org")
     del os.environ['no_proxy']

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
    def fetch(self, request, opener=None, summary=None):

        if not self.__enable_http:
            return (None, None)

        if opener is None:
            opener = OpenerDirector()
            opener.add_handler(HTTPDefaultErrorHandler())
            opener.add_handler(HTTPSHandler())

        t = time.clock()
        response = opener.open(request)
        body = response.read()
        t = timedelta(seconds=time.clock() - t)
        url = request.get_full_url()
        self.__context.get_logger().info('HTTP time: %s\n%s' % (t, url))

        if self.__log_http:
            log_dir = os.path.join(self.__context.get_config_dir(), 'http-log')
            makedirs(log_dir)
            log_file = os.path.join(log_dir,
                                    datetime.utcnow().strftime(
                                        '%Y-%m-%d-%H-%M-%S-%f'))
            if summary is not None:
                log_file += '-' + _safe_str(summary)
            fp = open(log_file, 'w')
            fp.write('\n\n'.join([
                request.get_full_url(),
                request.get_data() or 'No request data',
                body or 'No response body',
            ]))
            fp.close()

        return (response, body)

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
    def test_http_error(self):
        # XXX http_error_default
        # http errors are a special case
        o = OpenerDirector()
        meth_spec = [
            [("http_open", "error 302")],
            [("http_error_400", "raise"), "http_open"],
            [("http_error_302", "return response"), "http_error_303",
             "http_error"],
            [("http_error_302")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        class Unknown:
            def __eq__(self, other): return True

        req = Request("http://example.com/")
        r = o.open(req)
        assert len(o.calls) == 2
        calls = [(handlers[0], "http_open", (req,)),
                 (handlers[2], "http_error_302",
                  (req, Unknown(), 302, "", {}))]
        for expected, got in zip(calls, o.calls):
            handler, method_name, args = expected
            self.assertEqual((handler, method_name), got[:2])
            self.assertEqual(args, got[2])

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
    def test_processors(self):
        # *_request / *_response methods get called appropriately
        o = OpenerDirector()
        meth_spec = [
            [("http_request", "return request"),
             ("http_response", "return response")],
            [("http_request", "return request"),
             ("http_response", "return response")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        req = Request("http://example.com/")
        r = o.open(req)
        # processor methods are called on *all* handlers that define them,
        # not just the first handler that handles the request
        calls = [
            (handlers[0], "http_request"), (handlers[1], "http_request"),
            (handlers[0], "http_response"), (handlers[1], "http_response")]

        for i, (handler, name, args, kwds) in enumerate(o.calls):
            if i < 2:
                # *_request
                self.assertEqual((handler, name), calls[i])
                self.assertEqual(len(args), 1)
                self.assertIsInstance(args[0], Request)
            else:
                # *_response
                self.assertEqual((handler, name), calls[i])
                self.assertEqual(len(args), 2)
                self.assertIsInstance(args[0], Request)
                # response from opener.open is None, because there's no
                # handler that defines http_open to handle it
                self.assertTrue(args[1] is None or
                             isinstance(args[1], MockResponse))

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
    def test_handled(self):
        # handler returning non-None means no more handlers will be called
        o = OpenerDirector()
        meth_spec = [
            ["http_open", "ftp_open", "http_error_302"],
            ["ftp_open"],
            [("http_open", "return self")],
            [("http_open", "return self")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        req = Request("http://example.com/")
        r = o.open(req)
        # Second .http_open() gets called, third doesn't, since second returned
        # non-None.  Handlers without .http_open() never get any methods called
        # on them.
        # In fact, second mock handler defining .http_open() returns self
        # (instead of response), which becomes the OpenerDirector's return
        # value.
        self.assertEqual(r, handlers[2])
        calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
        for expected, got in zip(calls, o.calls):
            handler, name, args, kwds = got
            self.assertEqual((handler, name), expected)
            self.assertEqual(args, (req,))

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
def save_to_file(url: str, file_path: str, opener: OpenerDirector = None):
    def process(request_):
        with open(file_path, 'wb') as f:
            f.write(request_.read())

    if opener:
        with closing(opener.open(url)) as request:
            process(request)
    else:
        with closing(urlopen(url)) as request:
            process(request)

# 需要导入模块: from urllib.request import OpenerDirector [as 别名]
# 或者: from urllib.request.OpenerDirector import open [as 别名]
    def test_handler_order(self):
        o = OpenerDirector()
        handlers = []
        for meths, handler_order in [
            ([("http_open", "return self")], 500),
            (["http_open"], 0),
            ]:
            class MockHandlerSubclass(MockHandler): pass
            h = MockHandlerSubclass(meths)
            h.handler_order = handler_order
            handlers.append(h)
            o.add_handler(h)

        r = o.open("http://example.com/")
        # handlers called in reverse order, thanks to their sort order
        self.assertEqual(o.calls[0][0], handlers[1])
        self.assertEqual(o.calls[1][0], handlers[0])