Python urllib2.OpenerDirector类代码示例

    def test_processors(self):
        # *_request / *_response methods get called appropriately
        o = OpenerDirector()
        meth_spec = [
            [("http_request", "return request"),
             ("http_response", "return response")],
            [("http_request", "return request"),
             ("http_response", "return response")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        req = Request("http://example.com/")
        r = o.open(req)
        # processor methods are called on *all* handlers that define them,
        # not just the first handler that handles the request
        calls = [
            (handlers[0], "http_request"), (handlers[1], "http_request"),
            (handlers[0], "http_response"), (handlers[1], "http_response")]

        for i, (handler, name, args, kwds) in enumerate(o.calls):
            if i < 2:
                # *_request
                self.assertEqual((handler, name), calls[i])
                self.assertEqual(len(args), 1)
                self.assert_(isinstance(args[0], Request))
            else:
                # *_response
                self.assertEqual((handler, name), calls[i])
                self.assertEqual(len(args), 2)
                self.assert_(isinstance(args[0], Request))
                # response from opener.open is None, because there's no
                # handler that defines http_open to handle it
                self.assert_(args[1] is None or
                             isinstance(args[1], MockResponse))

    def test_http_error(self):
        # XXX http_error_default
        # http errors are a special case
        o = OpenerDirector()
        meth_spec = [
            [("http_open", "error 302")],
            [("http_error_400", "raise"), "http_open"],
            [("http_error_302", "return response"), "http_error_303",
             "http_error"],
            [("http_error_302")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        class Unknown:
            def __eq__(self, other): return True

        req = Request("http://example.com/")
        r = o.open(req)
        assert len(o.calls) == 2
        calls = [(handlers[0], "http_open", (req,)),
                 (handlers[2], "http_error_302",
                  (req, Unknown(), 302, "", {}))]
        for expected, got in zip(calls, o.calls):
            handler, method_name, args = expected
            self.assertEqual((handler, method_name), got[:2])
            self.assertEqual(args, got[2])

    def test_handled(self):
        # handler returning non-None means no more handlers will be called
        o = OpenerDirector()
        meth_spec = [
            ["http_open", "ftp_open", "http_error_302"],
            ["ftp_open"],
            [("http_open", "return self")],
            [("http_open", "return self")],
            ]
        handlers = add_ordered_mock_handlers(o, meth_spec)

        req = Request("http://example.com/")
        r = o.open(req)
        # Second .http_open() gets called, third doesn't, since second returned
        # non-None.  Handlers without .http_open() never get any methods called
        # on them.
        # In fact, second mock handler defining .http_open() returns self
        # (instead of response), which becomes the OpenerDirector's return
        # value.
        self.assertEqual(r, handlers[2])
        calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
        for expected, got in zip(calls, o.calls):
            handler, name, args, kwds = got
            self.assertEqual((handler, name), expected)
            self.assertEqual(args, (req,))

 def test_proxy_https(self):
     o = OpenerDirector()
     ph = urllib2.ProxyHandler(dict(https="proxy.example.com:3128"))
     o.add_handler(ph)
     meth_spec = [[("https_open", "return response")]]
     handlers = add_ordered_mock_handlers(o, meth_spec)
     req = Request("https://www.example.com/")
     self.assertEqual(req.get_host(), "www.example.com")
     r = o.open(req)
     self.assertEqual(req.get_host(), "proxy.example.com:3128")
     self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls])

 def test_basic_auth(self):
     opener = OpenerDirector()
     password_manager = MockPasswordManager()
     auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
     realm = "ACME Widget Store"
     http_handler = MockHTTPHandler(
         401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
     opener.add_handler(auth_handler)
     opener.add_handler(http_handler)
     self._test_basic_auth(opener, auth_handler, "Authorization",
                           realm, http_handler, password_manager,
                           "http://acme.example.com/protected",
                           "http://acme.example.com/protected",
                           )

 def test_proxy_no_proxy(self):
     os.environ['no_proxy'] = 'python.org'
     o = OpenerDirector()
     ph = urllib2.ProxyHandler(dict(http="proxy.example.com"))
     o.add_handler(ph)
     req = Request("http://www.perl.org/")
     self.assertEqual(req.get_host(), "www.perl.org")
     r = o.open(req)
     self.assertEqual(req.get_host(), "proxy.example.com")
     req = Request("http://www.python.org")
     self.assertEqual(req.get_host(), "www.python.org")
     r = o.open(req)
     self.assertEqual(req.get_host(), "www.python.org")
     del os.environ['no_proxy']

 def test_proxy_basic_auth(self):
     opener = OpenerDirector()
     ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128"))
     opener.add_handler(ph)
     password_manager = MockPasswordManager()
     auth_handler = urllib2.ProxyBasicAuthHandler(password_manager)
     realm = "ACME Networks"
     http_handler = MockHTTPHandler(
         407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
     opener.add_handler(auth_handler)
     opener.add_handler(http_handler)
     self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
                           realm, http_handler, password_manager,
                           "http://acme.example.com:3128/protected",
                           "proxy.example.com:3128",
                           )

    def test_handler_order(self):
        o = OpenerDirector()
        handlers = []
        for meths, handler_order in [
            ([("http_open", "return self")], 500),
            (["http_open"], 0),
            ]:
            class MockHandlerSubclass(MockHandler): pass
            h = MockHandlerSubclass(meths)
            h.handler_order = handler_order
            handlers.append(h)
            o.add_handler(h)

        r = o.open("http://example.com/")
        # handlers called in reverse order, thanks to their sort order
        self.assertEqual(o.calls[0][0], handlers[1])
        self.assertEqual(o.calls[1][0], handlers[0])

 def _make_request(self, url, method="GET", payload="", headers={}):
     self.opener = OpenerDirector()
     self.opener.add_handler(HTTPHandler())
     request = Request(url, headers=headers, data=payload.encode('utf-8'))
     request.get_method = lambda: method
     response = self.opener.open(request)
     response_code = getattr(response, 'code', -1)
     return (response, response_code)

    def test_badly_named_methods(self):
        # test work-around for three methods that accidentally follow the
        # naming conventions for handler methods
        # (*_open() / *_request() / *_response())

        # These used to call the accidentally-named methods, causing a
        # TypeError in real code; here, returning self from these mock
        # methods would either cause no exception, or AttributeError.

        from urllib2 import URLError

        o = OpenerDirector()
        meth_spec = [[("do_open", "return self"), ("proxy_open", "return self")], [("redirect_request", "return self")]]
        handlers = add_ordered_mock_handlers(o, meth_spec)
        o.add_handler(urllib2.UnknownHandler())
        for scheme in "do", "proxy", "redirect":
            self.assertRaises(URLError, o.open, scheme + "://example.com/")

    def __build_opener(self):
        """To prevent automatic redirects on 302 HTTP code."""
        from urllib2 import (OpenerDirector, ProxyHandler, UnknownHandler, HTTPHandler,
                             HTTPDefaultErrorHandler, HTTPRedirectHandler,
                             FTPHandler, FileHandler)

        opener = OpenerDirector()

        default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                           HTTPDefaultErrorHandler, HTTPRedirectHandler,
                           FTPHandler, FileHandler]

        if hasattr(httplib, 'HTTPS'):
            default_classes.append(urllib2.HTTPSHandler)

        for clazz in default_classes:
            opener.add_handler(clazz())

        return opener

    def __init__(self):
        OpenerDirector.__init__(self)

        self._ua_handlers = {}
        for scheme in (self.default_schemes+
                       self.default_others+
                       self.default_features):
            klass = self.handler_classes[scheme]
            self._ua_handlers[scheme] = klass()
        for handler in self._ua_handlers.itervalues():
            self.add_handler(handler)

        # Ensure correct default constructor args were passed to
        # HTTPRefererProcessor and HTTPEquivProcessor.  Yuck.
        if '_refresh' in self._ua_handlers:
            self.set_handle_refresh(True)
        if '_equiv' in self._ua_handlers:
            self.set_handle_equiv(True)

        # special case, requires extra support from mechanize.Browser
        self._handle_referer = True

    def fetch(self, request, opener=None, summary=None):

        if not self.__enable_http:
            return (None, None)

        if opener is None:
            opener = OpenerDirector()
            opener.add_handler(HTTPDefaultErrorHandler())
            opener.add_handler(HTTPSHandler())

        t = time.clock()
        response = opener.open(request)
        body = response.read()
        t = timedelta(seconds=time.clock() - t)
        url = request.get_full_url()
        self.__context.get_logger().info("HTTP time: %s\n%s" % (t, url))

        if self.__log_http:
            log_dir = os.path.join(self.__context.get_config_dir(), "http-log")
            makedirs(log_dir)
            log_file = os.path.join(log_dir, datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S-%f"))
            if summary is not None:
                log_file += "-" + _safe_str(summary)
            fp = open(log_file, "w")
            fp.write(
                "\n\n".join(
                    [request.get_full_url(), request.get_data() or "No request data", body or "No response body"]
                )
            )
            fp.close()

        return (response, body)

def build_opener(*handlers, **kw):
    """Create an opener object from a list of handlers.

    The opener will use several default handlers, including support
    for HTTP and FTP.

    If any of the handlers passed as arguments are subclasses of the
    default handlers, the default handlers will not be used.
    """
    import types
    def isclass(obj):
        return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__")

    opener = OpenerDirector()
    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
                       FTPHandler, FileHandler, HTTPErrorProcessor]
    check_classes = list(default_classes)
    check_classes.append(HTTPSContextHandler)
    skip = []
    for klass in check_classes:
        for check in handlers:
            if isclass(check):
                if issubclass(check, klass):
                    skip.append(klass)
            elif isinstance(check, klass):
                skip.append(klass)

    for klass in default_classes:
        if klass not in skip:
            opener.add_handler(klass())
            
    # Pick up SSL context from keyword settings
    ssl_context = kw.get('ssl_context')
    
    # Add the HTTPS handler with ssl_context
    if HTTPSContextHandler not in skip:
        opener.add_handler(HTTPSContextHandler(ssl_context))

    for h in handlers:
        if isclass(h):
            h = h()
        opener.add_handler(h)

    return opener

 def test_proxy_https_proxy_authorization(self):
     o = OpenerDirector()
     ph = urllib2.ProxyHandler(dict(https="proxy.example.com:3128"))
     o.add_handler(ph)
     https_handler = MockHTTPSHandler()
     o.add_handler(https_handler)
     req = Request("https://www.example.com/")
     req.add_header("Proxy-Authorization", "FooBar")
     req.add_header("User-Agent", "Grail")
     self.assertEqual(req.get_host(), "www.example.com")
     self.assertIsNone(req._tunnel_host)
     r = o.open(req)
     # Verify Proxy-Authorization gets tunneled to request.
     # httpsconn req_headers do not have the Proxy-Authorization header but
     # the req will have.
     self.assertNotIn(("Proxy-Authorization", "FooBar"), https_handler.httpconn.req_headers)
     self.assertIn(("User-Agent", "Grail"), https_handler.httpconn.req_headers)
     self.assertIsNotNone(req._tunnel_host)
     self.assertEqual(req.get_host(), "proxy.example.com:3128")
     self.assertEqual(req.get_header("Proxy-authorization"), "FooBar")