Python html5lib.parse方法代码示例

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def __init__(self, content, url, headers=None):
        # Determine if we have any encoding information in our headers
        encoding = None
        if headers and "Content-Type" in headers:
            content_type, params = cgi.parse_header(headers["Content-Type"])

            if "charset" in params:
                encoding = params['charset']

        self.content = content
        self.parsed = html5lib.parse(
            self.content,
            transport_encoding=encoding,
            namespaceHTMLElements=False,
        )
        self.url = url
        self.headers = headers 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def __init__(self, content, url, headers=None):
        # Determine if we have any encoding information in our headers
        encoding = None
        if headers and "Content-Type" in headers:
            content_type, params = cgi.parse_header(headers["Content-Type"])

            if "charset" in params:
                encoding = params['charset']

        self.content = content
        self.parsed = html5lib.parse(
            self.content,
            encoding=encoding,
            namespaceHTMLElements=False,
        )
        self.url = url
        self.headers = headers 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def _egg_info_matches(egg_info, canonical_name):
    # type: (str, str) -> Optional[str]
    """Pull the version part out of a string.

    :param egg_info: The string to parse. E.g. foo-2.1
    :param canonical_name: The canonicalized name of the package this
        belongs to.
    """
    try:
        version_start = _find_name_version_sep(egg_info, canonical_name) + 1
    except ValueError:
        return None
    version = egg_info[version_start:]
    if not version:
        return None
    return version 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def _determine_base_url(document, page_url):
    """Determine the HTML document's base URL.

    This looks for a ``<base>`` tag in the HTML document. If present, its href
    attribute denotes the base URL of anchor tags in the document. If there is
    no such tag (or if it does not have a valid href attribute), the HTML
    file's URL is used as the base URL.

    :param document: An HTML document representation. The current
        implementation expects the result of ``html5lib.parse()``.
    :param page_url: The URL of the HTML document.
    """
    for base in document.findall(".//base"):
        href = base.get("href")
        if href is not None:
            return href
    return page_url 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def iter_links(self):
        # type: () -> Iterable[Link]
        """Yields all links in the page"""
        document = html5lib.parse(
            self.content,
            transport_encoding=_get_encoding_from_headers(self.headers),
            namespaceHTMLElements=False,
        )
        base_url = _determine_base_url(document, self.url)
        for anchor in document.findall(".//a"):
            if anchor.get("href"):
                href = anchor.get("href")
                url = _clean_link(urllib_parse.urljoin(base_url, href))
                pyrequire = anchor.get('data-requires-python')
                pyrequire = unescape(pyrequire) if pyrequire else None
                yield Link(url, self.url, requires_python=pyrequire) 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def _determine_base_url(document, page_url):
    # type: (HTMLElement, str) -> str
    """Determine the HTML document's base URL.

    This looks for a ``<base>`` tag in the HTML document. If present, its href
    attribute denotes the base URL of anchor tags in the document. If there is
    no such tag (or if it does not have a valid href attribute), the HTML
    file's URL is used as the base URL.

    :param document: An HTML document representation. The current
        implementation expects the result of ``html5lib.parse()``.
    :param page_url: The URL of the HTML document.
    """
    for base in document.findall(".//base"):
        href = base.get("href")
        if href is not None:
            return href
    return page_url 

# 需要导入模块: from pip._vendor import html5lib [as 别名]
# 或者: from pip._vendor.html5lib import parse [as 别名]
def parse_links(page):
    # type: (HTMLPage) -> Iterable[Link]
    """
    Parse an HTML document, and yield its anchor elements as Link objects.
    """
    document = html5lib.parse(
        page.content,
        transport_encoding=page.encoding,
        namespaceHTMLElements=False,
    )

    url = page.url
    base_url = _determine_base_url(document, url)
    for anchor in document.findall(".//a"):
        link = _create_link_from_element(
            anchor,
            page_url=url,
            base_url=base_url,
        )
        if link is None:
            continue
        yield link