Python urlparse.urlparse方法代码示例

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def links(self):
        """Yields all links in the page"""
        for anchor in self.parsed.findall(".//a"):
            if anchor.get("href"):
                href = anchor.get("href")
                url = self.clean_link(urlparse.urljoin(self.base_url, href))

                # Determine if this link is internal. If that distinction
                #   doesn't make sense in this context, then we don't make
                #   any distinction.
                internal = None
                if self.api_version and self.api_version >= 2:
                    # Only api_versions >= 2 have a distinction between
                    #   external and internal links
                    internal = bool(anchor.get("rel")
                                and "internal" in anchor.get("rel").split())

                yield Link(url, self, internal=internal) 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def _get_content_type(url, session=None):
        """Get the Content-Type of the given url, using a HEAD request"""
        if session is None:
            session = PipSession()

        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
        if not scheme in ('http', 'https', 'ftp', 'ftps'):
            ## FIXME: some warning or something?
            ## assertion error?
            return ''

        resp = session.head(url, allow_redirects=True)
        resp.raise_for_status()

        return resp.headers.get("Content-Type", "") 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def explicit_rel_links(self, rels=('homepage', 'download')):
        """Yields all links with the given relations"""
        rels = set(rels)

        for anchor in self.parsed.findall(".//a"):
            if anchor.get("rel") and anchor.get("href"):
                found_rels = set(anchor.get("rel").split())
                # Determine the intersection between what rels were found and
                #   what rels were being looked for
                if found_rels & rels:
                    href = anchor.get("href")
                    url = self.clean_link(urlparse.urljoin(self.base_url, href))
                    yield Link(url, self, trusted=False) 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def scraped_rel_links(self):
        # Can we get rid of this horrible horrible method?
        for regex in (self._homepage_re, self._download_re):
            match = regex.search(self.content)
            if not match:
                continue
            href_match = self._href_re.search(self.content, pos=match.end())
            if not href_match:
                continue
            url = href_match.group(1) or href_match.group(2) or href_match.group(3)
            if not url:
                continue
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self, trusted=False, _deprecated_regex=True) 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def filename(self):
        _, netloc, path, _, _ = urlparse.urlsplit(self.url)
        name = posixpath.basename(path.rstrip('/')) or netloc
        assert name, ('URL %r produced no filename' % self.url)
        return name 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def path(self):
        return urlparse.urlsplit(self.url)[2] 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def url_without_fragment(self):
        scheme, netloc, path, query, fragment = urlparse.urlsplit(self.url)
        return urlparse.urlunsplit((scheme, netloc, path, query, None)) 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def __call__(self, req):
        parsed = urlparse.urlparse(req.url)

        # Get the netloc without any embedded credentials
        netloc = parsed.netloc.split("@", 1)[-1]

        # Set the url of the request to the url without any credentials
        req.url = urlparse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])

        # Use any stored credentials that we have for this netloc
        username, password = self.passwords.get(netloc, (None, None))

        # Extract credentials embedded in the url if we have none stored
        if username is None:
            username, password = self.parse_credentials(parsed.netloc)

        if username or password:
            # Store the username and password
            self.passwords[netloc] = (username, password)

            # Send the basic auth with this request
            req = HTTPBasicAuth(username or "", password or "")(req)

        # Attach a hook to handle 401 responses
        req.register_hook("response", self.handle_401)

        return req 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def handle_401(self, resp, **kwargs):
        # We only care about 401 responses, anything else we want to just
        #   pass through the actual response
        if resp.status_code != 401:
            return resp

        # We are not able to prompt the user so simple return the response
        if not self.prompting:
            return resp

        parsed = urlparse.urlparse(resp.url)

        # Prompt the user for a new username and password
        username = raw_input("User for %s: " % parsed.netloc)
        password = getpass.getpass("Password: ")

        # Store the new username and password to use for future requests
        if username or password:
            self.passwords[parsed.netloc] = (username, password)

        # Consume content and release the original connection to allow our new
        #   request to reuse the same one.
        resp.content
        resp.raw.release_conn()

        # Add our new username and password to the request
        req = HTTPBasicAuth(username or "", password or "")(resp.request)

        # Send our new request
        new_resp = resp.connection.send(req, **kwargs)
        new_resp.history.append(resp)

        return new_resp 

# 需要导入模块: from pip.backwardcompat import urlparse [as 别名]
# 或者: from pip.backwardcompat.urlparse import urlparse [as 别名]
def send(self, request, stream=None, timeout=None, verify=None, cert=None,
             proxies=None):
        parsed_url = urlparse.urlparse(request.url)

        # We only work for requests with a host of localhost
        if parsed_url.netloc.lower() != "localhost":
            raise InvalidURL("Invalid URL %r: Only localhost is allowed" %
                request.url)

        real_url = urlparse.urlunparse(parsed_url[:1] + ("",) + parsed_url[2:])
        pathname = url_to_path(real_url)

        resp = Response()
        resp.status_code = 200
        resp.url = real_url

        stats = os.stat(pathname)
        modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
        resp.headers = CaseInsensitiveDict({
            "Content-Type": mimetypes.guess_type(pathname)[0] or "text/plain",
            "Content-Length": stats.st_size,
            "Last-Modified": modified,
        })

        resp.raw = LocalFSResponse(open(pathname, "rb"))
        resp.close = resp.raw.close

        return resp