Python urllib.splittype函数代码示例

 def open(self, fullurl, data=None, method=None):
     """Use URLopener().open(file) instead of open(file, 'r')."""
     fullurl = unwrap(toBytes(fullurl))
     # percent encode url, fixing lame server errors for e.g, like space
     # within url paths.
     fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
     if self.tempcache and fullurl in self.tempcache:
         filename, headers = self.tempcache[fullurl]
         fp = open(filename, 'rb')
         return addinfourl(fp, headers, fullurl)
     urltype, url = splittype(fullurl)
     if not urltype:
         urltype = 'file'
     if urltype in self.proxies:
         proxy = self.proxies[urltype]
         urltype, proxyhost = splittype(proxy)
         host, selector = splithost(proxyhost)
         url = (host, fullurl) # Signal special case to open_*()
     else:
         proxy = None
     name = 'open_' + urltype
     self.type = urltype
     name = name.replace('-', '_')
     if not hasattr(self, name):
         if proxy:
             return self.open_unknown_proxy(proxy, fullurl, data)
         else:
             return self.open_unknown(fullurl, data)
     try:
         return getattr(self, name)(url, data, method)
     except socket.error, msg:
         raise IOError, ('socket error', msg), sys.exc_info()[2]

 def start_a(self, attrs):
     if self.get_object():           # expensive!
         self.get_object().anchor(attrs)
         return
     name = type = target = title = ''
     id = None
     has_key = attrs.has_key
     #
     href = string.strip(attrs.get("urn", ""))
     scheme, resturl = urllib.splittype(href)
     if scheme == "urn":
         scheme, resturl = urllib.splittype(resturl)
     if scheme not in ("doi", "hdl", "ietf"):
         # this is an unknown URN scheme or there wasn't a URN
         href = string.strip(attrs.get("href", ""))
     name = extract_keyword('name', attrs,
                            conv=grailutil.conv_normstring)
     if has_key('type'): type = string.lower(attrs['type'] or '')
     if has_key('target'): target = attrs['target']
     if has_key('id'): id = attrs['id']
     self.anchor_bgn(href, name, type, target, id)
     # Delay this at least a little, since we don't want to add the title
     # to the history until the last possible moment.  We need a non-history
     # way to do this; a resources database would be much better.
     if has_key('title'):
         title = string.join(string.split(attrs['title'] or ''))
         if title:
             url = self.context.get_baseurl(
                 string.joinfields(string.split(href), ''))
             old_title, when = self.app.global_history.lookup_url(url)
             if not old_title:
                 # Only do this if there's not already a title in the
                 # history.  If the URL wasn't in the history, it will
                 # be given a timestamp, which is bad. ;-(
                 self.app.global_history.set_title(url, title)

  def __init__(self, uri, transport=None, encoding=None, verbose=0,
               allow_none=0, use_datetime=0):
    urltype, _ = urllib.splittype(uri)
    if urltype not in ('http', 'https'):
      raise IOError('unsupported JSON-RPC protocol')

    _base.ServerProxy.__init__(self, uri, transport, encoding, verbose,
                               allow_none, use_datetime)
    transport_type, uri = urllib.splittype(uri)
    if transport is None:
      if transport_type == 'https':
        transport = SafeTransport(use_datetime=use_datetime)
      else:
        transport = Transport(use_datetime=use_datetime)
    self.__transport = transport

    def do_request_(self, request):
        host = request.get_host()
        if not host:
            raise URLError('no host given')

        if request.has_data():  # POST
            data = request.get_data()
            if not request.has_header('Content-type'):
                request.add_unredirected_header(
                    'Content-type',
                    'application/x-www-form-urlencoded')
            if not request.has_header('Content-length'):
                request.add_unredirected_header(
                    'Content-length', '%d' % len(data))

        scheme, sel = splittype(request.get_selector())
        sel_host, sel_path = splithost(sel)
        if not request.has_header('Host'):
            request.add_unredirected_header('Host', sel_host or host)
        for name, value in self.parent.addheaders:
            name = name.capitalize()
            if not request.has_header(name):
                request.add_unredirected_header(name, value)

        return request

 def parse_callback_url(self, callback_url):
     proto, rest = urllib.splittype(callback_url)
     host, rest = urllib.splithost(rest)
     host, port = urllib.splitport(host)
     if not port:
         port = 443
     return host, port

    def getpage(self, url_pair):
        # Incoming argument name is a (URL, fragment) pair.
        # The page may have been cached in the name_table variable.
        url, fragment = url_pair
        if self.name_table.has_key(url):
            return self.name_table[url]

        scheme, path = urllib.splittype(url)
        if scheme in ('mailto', 'news', 'javascript', 'telnet'):
            self.note(1, " Not checking %s URL" % scheme)
            return None
        isint = self.inroots(url)

        # Ensure that openpage gets the URL pair to
        # print out its error message and record the error pair
        # correctly.
        if not isint:
            if not self.checkext:
                self.note(1, " Not checking ext link")
                return None
            f = self.openpage(url_pair)
            if f:
                self.safeclose(f)
            return None
        text, nurl = self.readhtml(url_pair)

        if nurl != url:
            self.note(1, " Redirected to %s", nurl)
            url = nurl
        if text:
            return Page(text, url, maxpage=self.maxpage, checker=self)

 def request(self, method, url, body=None, headers={}):
     """
     Make CONNECT request to proxy.
     """
         
     proto, rest = urllib.splittype(url)
     if proto is None:
         raise ValueError, "unknown URL type: %s" % url
         
     # Get hostname.
     host = urllib.splithost(rest)[0]
     
     # Get port of one
     host, port = urllib.splitport(host)
     
     # When no port use hardcoded.
     if port is None:
         try:
             port = self._ports[proto]
         except KeyError:
             raise ValueError, "unknown protocol for: %s" % url
     
     # Remember.        
     self._real_host = host
     self._real_port = port
     
     # Remember auth if there.
     if headers.has_key("Proxy-Authorization"):
         self._proxy_authorization = headers["Proxy-Authorization"]
         del headers["Proxy-Authorization"]
     else:
         self._proxy_authorization = None
     
     httplib.HTTPConnection.request(self, method, url, body, headers)

def _add_proxy(session, config):
    if None in (config.proxy_url, config.proxy_port):
        return

    # Set session.proxies according to given url and port
    protocol, remainder = urllib.splittype(config.proxy_url)
    host, remainder = urllib.splithost(remainder)
    url = ':'.join((host, str(config.proxy_port)))

    if config.proxy_username:
        password_part = config.get('proxy_password', '') and ':%s' % config.proxy_password
        auth = config.proxy_username + password_part
        auth = urllib.quote(auth, safe=':')
        url = '@'.join((auth, url))

    session.proxies['https'] = '://'.join((protocol, url))
    session.proxies['http'] = '://'.join((protocol, url))

    # Set session.auth if proxy username is specified
    if config.proxy_username is not None:
        proxy_password = config.get('proxy_password', '')
        if None in (config.basic_auth_username, config.basic_auth_password):
            # bz 1021662 - Proxy authentiation using username and password in session.proxies urls
            # does not setup correct headers in the http download request because of a bug in
            # urllib3. This is an alternate approach which sets up the headers correctly.
            session.auth = requests.auth.HTTPProxyAuth(config.proxy_username, proxy_password)
        else:
            # The approach mentioned above works well except when a basic user authentication is
            # used, along with the proxy authentication. Therefore, we define and use a custom class
            # which inherits AuthBase class provided by the requests library to add the headers
            # correctly.
            session.auth = HTTPBasicWithProxyAuth(config.basic_auth_username,
                                                  config.basic_auth_password,
                                                  config.proxy_username,
                                                  proxy_password)

def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and components[i - 1] not in ('', '..'):
            del components[i - 1:i + 1]
            i = i - 1
        elif components[i] == '' and i > 0 and components[i - 1] != '':
            del components[i]
        else:
            i = i + 1

    if not components[0]:
        rv = ':'.join(components[1:])
    else:
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1

        rv = ':' + ':'.join(components)
    return urllib.unquote(rv)

    def __init__(self, uri, transport=None, encoding=None, verbose=0, auth_username=None, auth_password=None):
        # establish a "logical" server connection

        # get the url
        import urllib

        type, uri = urllib.splittype(uri)
        if type:
            if type not in ("http", "https"):
                raise IOError, "unsupported XML-RPC protocol"
            self.__host, self.__handler = urllib.splithost(uri)
            if not self.__handler:
                self.__handler = "/RPC2"

            if transport is None:
                if type == "https":
                    transport = SafeTransport()
                else:
                    transport = Transport()
        else:
            self.__host = uri
            transport = RawTransport()

        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose

        self.__username = auth_username
        self.__password = auth_password

    def __init__(self, uri, username=None, password=None,
                 verify=False, sp=None, sp_kwargs=None):
        self.uri = self._transform_uri(uri)  # : From X{__init__(self, url)}

        self.username = username
        self.password = password

        self.scheme = urllib.splittype(self.uri)[0]

        if sp:
            self.sp = sp
        elif self.scheme in ['http', 'https']:
            self.sp = HTTPServerProxy
        elif self.scheme == 'scgi':
            self.sp = SCGIServerProxy
        else:
            raise NotImplementedError()

        self.sp_kwargs = sp_kwargs or {}

        self.torrents = []  # : List of L{Torrent} instances
        self._rpc_methods = []  # : List of rTorrent RPC methods
        self._torrent_cache = []
        self._client_version_tuple = ()

        if verify is True:
            self._verify_conn()

 def processRequest(self,method=None,url=None,data="",headers={}):
     conf = desktop.Config()
     if not conf['proxy']:
         self.proxy_host = None
         self.proxy_port = None
     else:
         self.proxy_host = conf['proxy']['proxy']
         self.proxy_port = conf['proxy']['proxy_port']
     socket.setdefaulttimeout(self.http_timeout)
     (protocol,resource) = urllib.splittype(url)
     (hostport,path) = urllib.splithost(resource)
     connexion = None
     if protocol.lower() == "http":
         (host,port) = urllib.splitnport(hostport, 80)
         import httplib
         if self.proxy_host != None and self.proxy_port != None :
             connexion = HTTPConnection(self.proxy_host, self.proxy_port, timeout=self.http_timeout)
             path = url
         else:
             connexion = HTTPConnection(host, port, timeout=self.http_timeout)
     elif protocol.lower() == "https" :
         (host,port) = urllib.splitnport(hostport, 443)
         connexion = HTTPSConnection(host, port)
         if self.proxy_host != None and self.proxy_port != None :
             connexion.http_proxy = [self.proxy_host,
                                     self.proxy_port]
     else:
         assert False, "Unhandled Protocol, please use HTTP or HTTPS"
     
         
     connexion.connect()
     connexion.request(method, path, body=data, headers=headers)
     response = connexion.getresponse()
     
     return response

    def init_server(self, myuri):
        #Borrowed the following from rpcServer.py
        #rpclib.Server.__init__(self, uri, transport=self.rpc_args['transport'], encoding=self.rpc_args['encoding'], verbose=self.rpc_args['verbose'],\
        #                              proxy=self.rpc_args['proxy'], username=self.rpc_args['username'],\
        #                              password=self.rpc_args['password'], refreshCallback=self.rpc_args['refreshCallback'],\
        #                              progressCallback=self.rpc_args['progressCallback'])
        self._uri = myuri
        typ, uri = urllib.splittype(self._uri)
        typ = typ.lower()
        if typ not in ("http", "https"):
            raise InvalidRedirectionError(
                "Redirected to unsupported protocol %s" % typ)

        self._host, self._handler = urllib.splithost(uri)
        self._orig_handler = self._handler
        self._type = typ
        if not self._handler:
            self._handler = self.rpc_handler
        self._allow_redirect = 1
        del self._transport
        self._transport = self.default_transport(typ, self._proxy,
                                             self._username, self._password)
        self.set_progress_callback(self._progressCallback)
        self.set_refresh_callback(self._refreshCallback)
        self.set_buffer_size(self._bufferSize)
        self.setlang(self._lang)

        if self._trusted_cert_files != [] and \
            hasattr(self._transport, "add_trusted_cert"):
            for certfile in self._trusted_cert_files:
                self._transport.add_trusted_cert(certfile)

    def _get_real_authority(self):
        """
        Return the authority specification of the originally requested URL.

        The return value is a string of the form <host>:<port>.

        """

        url = self._proxy_request.get_selector()

        proto, rest = urllib.splittype(url)
        if proto is None:
            raise ValueError("unknown URL type: %s" % url)

        # Get the host and port specification
        host, rest = urllib.splithost(rest)
        host, port = urllib.splitport(host)

        # If port is not defined, then try to get it from the protocol.
        if port is None:
            try:
                port = self._ports[proto]
            except KeyError:
                raise ValueError("unknown protocol for: %s" % url)

        return '%s:%d' % (host, port)

    def compile(self):
        """Validate the user submitted url address at compile stage.

        The url address will be tested with the configured regex patterns
        loaded from :attr:`BaseHost.compiler_params`.
        Refer to :ref:`hwnetapi` for more details about the rules.
        """
        if self.config['urlrule']:
            p = re.compile(self.config['urlrule'])
            if not p.match(self.config['remote_addr']):
                raise NetApiAddressRejected(compile_error=lazy_gettext(
                    'Address "%(url)s" does not match pattern "%(rule)s"',
                    url=self.config['remote_addr'], rule=self.config['urlrule']
                ))
        if self.config['iprule']:
            domain = urllib.splitport(
                urllib.splithost(
                    urllib.splittype(self.config['remote_addr'])[1]
                )[0]
            )[0]
            # get ip from domain
            try:
                ipaddr = socket.gethostbyname(domain)
            except Exception:
                logger.exception(
                    'Could not get ip address for domain "%s".' % domain)
                ipaddr = '<invalid>'
            # ip not match, skip
            p = re.compile(self.config['iprule'])
            if not p.match(ipaddr):
                raise NetApiAddressRejected(compile_error=lazy_gettext(
                    'IP address "%(ip)s" does not match pattern "%(rule)s"',
                    ip=ipaddr, rule=self.config['iprule']
                ))