Python request.build_opener函数代码示例

def _create_opener():
    """
    Creates an opener for the internet.

    It also attaches the :class:`CachingRedirectHandler` to the opener and
    sets its User-agent to ``Mozilla/5.0``.

    If the Network Proxy settings are set and recognized, it creates the
    opener and attaches the proxy_handler to it. The opener is tested and
    returned if the test passes.

    If the test fails an opener without the proxy settings is created instead
    and is returned instead.
    """
    use_proxy = False
    proxy_handler = None

    if NETWORK_PROXY_TYPE == 'http':
        use_proxy = True
        proxyurl = _get_http_proxy_url()
        proxy_handler = ProxyHandler({'http': proxyurl,
                                      'https': proxyurl})
    if use_proxy:
        openr = build_opener(HTTPHandler(), HTTPSHandler(),
                             proxy_handler, CachingRedirectHandler)
    else:
        openr = build_opener(HTTPSHandler(), HTTPSHandler(),
                             CachingRedirectHandler)
    openr.addheaders = [('User-agent', 'Mozilla/5.0')]
    global _internet_connected
    _internet_connected = _test_opener(openr)
    return openr

def putDataUrllib2(url,payload,timeout=900,logger=None):
    """
    Wrapper method for urllib2 that supports PUTs to a url.

    Parameters
    ----------
    url : `string`
        Ex: 'https://dqsegdb5.phy.syr.edu/L1/DMT-SCIENCE/1'
    payload : `string`
        JSON formatted string

    """
    socket.setdefaulttimeout(timeout)
    #BEFORE HTTPS: opener = urllib2.build_opener(urllib2.HTTPHandler)
    if urlparse(url).scheme == 'https':
        opener=urllib_request.build_opener(HTTPSClientAuthHandler)
    else:
        opener = urllib_request.build_opener(urllib_request.HTTPHandler)
    request = urllib_request.Request(url, data=payload)
    request.add_header('Content-Type', 'JSON')
    request.get_method = lambda: 'PUT'
    if logger:
        logger.debug("Beginning url call: %s" % url)
    try:
        urlreturned = opener.open(request)
    except urllib_error.HTTPError as e:
        handleHTTPError("PUT",url,e)
        ##print(e.read())
        #if int(e.code)==404:
        #    print("Flag does not exist in database yet for url: %s" % url)
        #else:
        #    print("Warning: Issue accessing url: %s" % url)
        #    print("Code: ")
        #    print(e.code)
        #    print("Message: ")
        #    print(e.msg)
        #    #print(e.reason)
        #    #print(url)
        #    print("May be handled cleanly by calling instance: otherwise will result in an error.")
        ##print(e.reason)
        ##print(urlreturned)
        raise
    except urllib_error.URLError as e:
        #print(e.read())
        warnmsg="Warning: Issue accessing url: %s" % url
        warnmsg+="; "
        warnmsg+=str(e.reason)
        warnmsg+="; "
        warnmsg+="May be handled cleanly by calling instance: otherwise will result in an error."
        warn(warnmsg)
        raise
    if logger:
        logger.debug("Completed url call: %s" % url)
    return url

def open_no_proxy(*args, **kwargs):
    # NOTE(jamespage):
    # Deal with more secure certification chain verficiation
    # introduced in python 2.7.9 under PEP-0476
    # https://github.com/python/peps/blob/master/pep-0476.txt
    if hasattr(ssl, "_create_unverified_context"):
        opener = urlrequest.build_opener(
            urlrequest.ProxyHandler({}), urlrequest.HTTPSHandler(context=ssl._create_unverified_context())
        )
    else:
        opener = urlrequest.build_opener(urlrequest.ProxyHandler({}))
    return opener.open(*args, **kwargs)

 def _get_opener(self):
     if not self.opener:
         if (CONF.dashboard.disable_ssl_certificate_validation and
            self._ssl_default_context_supported()):
             ctx = ssl.create_default_context()
             ctx.check_hostname = False
             ctx.verify_mode = ssl.CERT_NONE
             self.opener = request.build_opener(
                 request.HTTPSHandler(context=ctx),
                 request.HTTPCookieProcessor())
         else:
             self.opener = request.build_opener(
                 request.HTTPCookieProcessor())
     return self.opener

def url_get(base_url, password_mgr=None, pathspec=None, params=None):
    """Make contact with the uri provided and return any contents."""
    # Uses system proxy settings if they exist.
    proxy = urlrequest.ProxyHandler()
    if password_mgr is not None:
        auth = urlrequest.HTTPDigestAuthHandler(password_mgr)
        urlopener = urlrequest.build_opener(proxy, auth)
    else:
        urlopener = urlrequest.build_opener(proxy)
    urlrequest.install_opener(urlopener)
    full_url = build_url(base_url, pathspec=pathspec, params=params)
    response = urlopener.open(full_url)
    content = response.read()
    response.close()
    return content

def func(args):
    if args.outputdocument:
        fdout = open(args.outputdocument, 'w')
    else:
        fdout = sys.stdout

    if args.useragent:
        useragent = args.useragent
    else:
        useragent = 'Cmdix/' + cmdix.__version__

    opener = build_opener()
    opener.addheaders = [('User-agent', useragent)]

    for url in args.url:
        try:
            fdin = opener.open(url)
        except HTTPError as e:
            exception.StdErrException(
                "HTTP error opening {0}: {1}".format(url, e))

        length = int(fdin.headers['content-length'])
        print("Getting {0} bytes from {1}...".format(length, url))

        shutil.copyfileobj(fdin, fdout)
        print("Done")

 def __init__(self, url, keystone_url, credentials, **kwargs):
     logger.info('Initiate HTTPClient with url %s', url)
     self.url = url
     self.keystone_url = keystone_url
     self.creds = dict(credentials, **kwargs)
     self.keystone = None
     self.opener = request.build_opener(request.HTTPHandler)

 def open(self, proxy=None, cache=True):
     if self.local:
         self._last_update_time = os.stat(self.url).st_mtime
         return open(self.url)
     elif cache and os.path.exists(self.cache):
         self._last_update_time = os.stat(self.cache).st_mtime
         return open(self.cache)
     else:
         r = Request(self.url)
         if proxy or self.proxy:
             proxy = proxy if proxy else self.proxy
             opener = build_opener(SocksiPyHandler(
                 socks.PROXY_TYPES[proxy['type'].upper()],
                 proxy['ip'],
                 proxy['port'],
             ))
             data_io = opener.open(r)
         else:
             data_io = urlopen(r)
         data = data_io.read()
         if self.is_base64(data):
             logger.debug('BASE64 decode...')
             data = base64.b64decode(data)
         if not isinstance(data, str):
             data = data.decode('utf-8')
         if self.cache:
             with open(self.cache, 'w') as f:
                 f.write(data)
             self._last_update_time = os.stat(self.cache).st_mtime
             return open(self.cache)
         self._last_update_time = time.time()
         return StringIO(data)

    def user_login(self, username, password):
        self.opener = request.build_opener(request.HTTPCookieProcessor())
        response = self.opener.open(CONF.dashboard.dashboard_url).read()

        # Grab the CSRF token and default region
        parser = HorizonHTMLParser()
        parser.feed(response)

        # construct login url for dashboard, discovery accommodates non-/ web
        # root for dashboard
        login_url = parse.urljoin(CONF.dashboard.dashboard_url, parser.login)

        # Prepare login form request
        req = request.Request(login_url)
        req.add_header("Content-type", "application/x-www-form-urlencoded")
        req.add_header("Referer", CONF.dashboard.dashboard_url)

        # Pass the default domain name regardless of the auth version in order
        # to test the scenario of when horizon is running with keystone v3
        params = {
            "username": username,
            "password": password,
            "region": parser.region,
            "domain": CONF.auth.default_credentials_domain_name,
            "csrfmiddlewaretoken": parser.csrf_token,
        }
        self.opener.open(req, parse.urlencode(params))

def srtm_login_or_skip(monkeypatch):
    import os
    try:
        srtm_username = os.environ['SRTM_USERNAME']
    except KeyError:
        pytest.skip('SRTM_USERNAME environment variable is unset.')
    try:
        srtm_password = os.environ['SRTM_PASSWORD']
    except KeyError:
        pytest.skip('SRTM_PASSWORD environment variable is unset.')

    from six.moves.urllib.request import (HTTPBasicAuthHandler,
                                          HTTPCookieProcessor,
                                          HTTPPasswordMgrWithDefaultRealm,
                                          build_opener)
    from six.moves.http_cookiejar import CookieJar

    password_manager = HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password(
        None,
        "https://urs.earthdata.nasa.gov",
        srtm_username,
        srtm_password)
    cookie_jar = CookieJar()
    opener = build_opener(HTTPBasicAuthHandler(password_manager),
                          HTTPCookieProcessor(cookie_jar))

    monkeypatch.setattr(cartopy.io, 'urlopen', opener.open)

def set_proxy(proxy, user=None, password=''):
    """
    Set the HTTP proxy for Python to download through.

    If ``proxy`` is None then tries to set proxy from environment or system
    settings.

    :param proxy: The HTTP proxy server to use. For example:
        'http://proxy.example.com:3128/'
    :param user: The username to authenticate with. Use None to disable
        authentication.
    :param password: The password to authenticate with.
    """
    from nltk import compat

    if proxy is None:
        # Try and find the system proxy settings
        try:
            proxy = getproxies()['http']
        except KeyError:
            raise ValueError('Could not detect default proxy settings')

    # Set up the proxy handler
    proxy_handler = ProxyHandler({'https': proxy, 'http': proxy})
    opener = build_opener(proxy_handler)

    if user is not None:
        # Set up basic proxy authentication if provided
        password_manager = HTTPPasswordMgrWithDefaultRealm()
        password_manager.add_password(realm=None, uri=proxy, user=user, passwd=password)
        opener.add_handler(ProxyBasicAuthHandler(password_manager))
        opener.add_handler(ProxyDigestAuthHandler(password_manager))

    # Overide the existing url opener
    install_opener(opener)

    def _set_cookies(self, src):
        '''
        function that returns a urllib2 opener for retrieving data from *src*

        input:
            *src* : 'asos' or 'wunderground' or 'wunder_nonairport'
        '''
        jar = http_cookiejar.CookieJar()
        handler = request.HTTPCookieProcessor(jar)
        opener = request.build_opener(handler)
        try:
            if src.lower() == 'wunderground':
                url1 = 'http://www.wunderground.com/history/airport/%s/2011/12/4/DailyHistory.html?' % self.sta_id
                url2 = 'http://www.wunderground.com/cgi-bin/findweather/getForecast?setpref=SHOWMETAR&value=1'
                url3 = 'http://www.wunderground.com/history/airport/%s/2011/12/4/DailyHistory.html?&&theprefset=SHOWMETAR&theprefvalue=1&format=1' % self.sta_id

                opener.open(url1)
                opener.open(url2)
                opener.open(url3)

            elif src.lower() == 'asos':
                url = 'ftp://ftp.ncdc.noaa.gov/pub/data/asos-fivemin/'
                opener.open(url)

            elif src.lower() == 'wunder_nonairport':
                url = 'http://www.wunderground.com/weatherstation/WXDailyHistory.asp?ID=MEGKO3&day=1&year=2013&month=1&graphspan=day&format=1'
                opener.open(url)

        except error.URLError:
            print(('connection to %s not available. working locally' % src))

        return opener

    def __init__(self, url, username=None, password=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
        super().__init__(url, username, password, timeout)

        # Setting ssl key verificationas false
        context = ssl._create_stdlib_context(check_hostname=False)
        unverified_handler = HTTPSHandler(context=context, check_hostname=False)
        install_opener(build_opener(unverified_handler))

def _read_from_url(url):
    """Reads data from *url* with an HTTP *GET*.

    This function supports fetching from resources which use basic HTTP auth as
    laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs.

    .. seealso:

       https://www.ietf.org/rfc/rfc1738.txt

    :param url: URL of an HTTP resource
    :type url: ``str``

    :return: data read from resource described by *url*
    :rtype: ``file``-like object
    """
    url, username, password = _strip_basic_auth(url)
    if username is not None and password is not None:
        # case: url contains basic auth creds
        password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(None, url, username, password)
        handler = request.HTTPBasicAuthHandler(password_mgr)
        opener = request.build_opener(default_handlers + [handler])
    else:
        opener = default_opener

    return opener.open(url)

 def __init__(self, settings, address, handler_class, backend_port, username):
     # type: (CtlSettings, Tuple[str, int], Type[BaseHTTPRequestHandler], int, str) -> None
     self.backend_port = backend_port
     self.user_auth_header = settings.user_auth_header
     self.username = username
     self.opener = build_opener(NoRedirectHandler)
     super(ProxyServer, self).__init__(address, handler_class)