Python urllib.parse方法代码示例

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def resolve_reference_http(cls, design_uri):
        """Retrieve design documents from http/https endpoints.

        Return a byte array of the response content. Support unsecured or
        basic auth

        :param design_uri: Tuple as returned by urllib.parse for the design reference
        """
        if design_uri.username is not None and design_uri.password is not None:
            response = requests.get(
                design_uri.geturl(),
                auth=(design_uri.username, design_uri.password),
                timeout=get_client_timeouts())
        else:
            response = requests.get(
                design_uri.geturl(), timeout=get_client_timeouts())

        return response.content 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def resolve_reference_ucp(cls, design_uri):
        """Retrieve artifacts from a Airship service endpoint.

        Return a byte array of the response content. Assumes Keystone
        authentication required.

        :param design_uri: Tuple as returned by urllib.parse for the design reference
        """
        ks_sess = KeystoneUtils.get_session()
        (new_scheme, foo) = re.subn(r'^[^+]+\+', '', design_uri.scheme)
        url = urllib.parse.urlunparse(
            (new_scheme, design_uri.netloc, design_uri.path, design_uri.params,
             design_uri.query, design_uri.fragment))
        LOG.debug("Calling Keystone session for url %s" % str(url))
        resp = ks_sess.get(url, timeout=get_client_timeouts())
        if resp.status_code >= 400:
            raise errors.InvalidDesignReference(
                "Received error code for reference %s: %s - %s" %
                (url, str(resp.status_code), resp.text))
        return resp.content 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def resolveEntity(self, publicId, systemId):
        assert systemId is not None
        source = DOMInputSource()
        source.publicId = publicId
        source.systemId = systemId
        source.byteStream = self._get_opener().open(systemId)

        # determine the encoding if the transport provided it
        source.encoding = self._guess_media_encoding(source)

        # determine the base URI is we can
        import posixpath, urllib.parse
        parts = urllib.parse.urlparse(systemId)
        scheme, netloc, path, params, query, fragment = parts
        # XXX should we check the scheme here as well?
        if path and not path.endswith("/"):
            path = posixpath.dirname(path) + "/"
            parts = scheme, netloc, path, params, query, fragment
            source.baseURI = urllib.parse.urlunparse(parts)

        return source 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def bot_is_allowed(text, user):
    """
    Taken from https://en.wikipedia.org/wiki/Template:Bots
    For bot exclusion compliance.
    """
    user = user.lower().strip()
    text = mwparserfromhell.parse(text)
    for tl in text.filter_templates():
        if tl.name in ('bots', 'nobots'):
            break
    else:
        return True
    for param in tl.params:
        bots = [x.lower().strip() for x in param.value.split(",")]
        if param.name == 'allow':
            if ''.join(bots) == 'none': return False
            for bot in bots:
                if bot in (user, 'all'):
                    return True
        elif param.name == 'deny':
            if ''.join(bots) == 'none': return True
            for bot in bots:
                if bot in (user, 'all'):
                    return False
    return True 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def __init__(self,
                 name,
                 base_url,
                 ticker_field,
                 field_dict=None,
                 doc_link=None):
        # field dict includes all fields to be passed to the URL
        # for example, for Alphavantage
        # name = 'Alphavantage_digital'
        # base-url = 'https://www.alphavantage.co/query'
        # ticker_field = 'symbol'
        # field_dict = {'function': 'DIGITAL_CURRENCY_DAILY',
        #               'market': 'CNY',
        #               'apikey': 'demo')
        # doc_link = 'https://www.alphavantage.co/documentation/'
        # parse_dict = {'open' : '1a. open (USD)', ...}
        self.name = name.lower()
        self.base_url = base_url
        self.ticker_field = ticker_field
        self.field_dict = field_dict
        self.doc_link = doc_link
        if self.field_dict is not None:
            self.url_args = "&" + urllib.parse.urlencode(field_dict)
        self.errors = [] 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def _parse_cache_control(headers):
    retval = {}
    if "cache-control" in headers:
        parts = headers["cache-control"].split(",")
        parts_with_args = [
            tuple([x.strip().lower() for x in part.split("=", 1)])
            for part in parts
            if -1 != part.find("=")
        ]
        parts_wo_args = [
            (name.strip().lower(), 1) for name in parts if -1 == name.find("=")
        ]
        retval = dict(parts_with_args + parts_wo_args)
    return retval


# Whether to use a strict mode to parse WWW-Authenticate headers
# Might lead to bad results in case of ill-formed header value,
# so disabled by default, falling back to relaxed parsing.
# Set to true to turn on, usefull for testing servers. 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def _prepare_url(self, url, *args, **kwargs):
        # TODO: Move this out as a helper and unit-test it directly?
        b = urllib.parse.urlsplit(self._base_url)
        u = urllib.parse.urlsplit(url.format(*args))

        if u.scheme or u.netloc:
            scheme, netloc, path = u.scheme, u.netloc, u.path
            query = urllib.parse.parse_qsl(u.query, keep_blank_values=True)
        else:
            scheme, netloc = b.scheme, b.netloc
            path = os.path.normpath(os.path.join(b.path, u.path))
            query = urllib.parse.parse_qsl(b.query, keep_blank_values=True)
            query.extend(urllib.parse.parse_qsl(u.query, keep_blank_values=True))

        for key, value in kwargs.items():
            query.append((key, value))

        encoded_query = urllib.parse.urlencode(dict(query))
        return urllib.parse.urlunsplit((scheme, netloc, path, encoded_query, "")) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def filename_from_url(url: QUrl) -> typing.Optional[str]:
    """Get a suitable filename from a URL.

    Args:
        url: The URL to parse, as a QUrl.

    Return:
        The suggested filename as a string, or None.
    """
    if not url.isValid():
        return None
    pathname = posixpath.basename(url.path())
    if pathname:
        return pathname
    elif url.host():
        return url.host() + '.html'
    else:
        return None 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def serialize(self, fields=None, exclude=None):
        """
        serialize to a different form than used by the internal class structure
        used to append additional values (like parent ontology properties) that
        internal objects (like models.Nodes) don't support
        """
        serializer = JSONSerializer()
        serializer.geom_format = "geojson"
        obj = serializer.handle_model(self)
        ordered_cards = self.get_ordered_cards()
        ret = JSONSerializer().serializeToPython(obj)
        ret["cards"] = ordered_cards
        try:
            bounds = json.loads(ret["bounds"])
            ret["bounds"] = bounds
            if bounds["type"] == "MultiPolygon":
                singlepart = GeoUtils().convert_multipart_to_singlepart(bounds)
                ret["bounds"] = singlepart
        except TypeError as e:
            print("Could not parse", ret["bounds"], e)
        return ret 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def parse(self, limit=None):
        """
        Override Source.parse()
        Parses version and interaction information from CTD
        Args:
        :param limit (int, optional) limit the number of rows processed
        Returns:
        :return None
        """
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)

        LOG.info("Parsing files...")

        if self.test_only:
            self.test_mode = True

        self.geno = Genotype(self.graph)
        self.pathway = Pathway(self.graph)

        src_key = 'chemical_disease_associations'
        self._parse_ctd_file(limit, src_key)

        # self._parse_ctd_file(limit, 'gene_pathway')
        # self._parse_ctd_file(limit, 'gene_disease') 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def _url_status(url):
    parse_obj = urllib.parse.urlparse(url)

    timer = 1
    for i in range(6):
        try:
            connection = http.client.HTTPConnection(parse_obj.netloc)
            connection.request('HEAD', parse_obj.path)
            break
        except Exception as e:
            print(url, e, 'sleep', timer)
            time.sleep(timer)
            timer *= 2
    else:
        return e

    response = connection.getresponse()
    connection.close()
    return response.status 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def extract_credentials(url):
    """Extract authentication (user name and password) credentials from the
    given URL.

    >>> extract_credentials('http://localhost:5984/_config/')
    ('http://localhost:5984/_config/', None)
    >>> extract_credentials('http://joe:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe', 'secret'))
    >>> extract_credentials('http://joe%40example.com:secret@localhost:5984/_config/')
    ('http://localhost:5984/_config/', ('joe@example.com', 'secret'))
    """
    parts = urllib.parse.urlsplit(url)
    netloc = parts[1]
    if '@' in netloc:
        creds, netloc = netloc.split('@')
        credentials = tuple(urllib.parse.unquote(i)
                            for i in creds.split(':'))
        parts = list(parts)
        parts[1] = netloc
    else:
        credentials = None
    return urllib.parse.urlunsplit(parts), credentials 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def _Parse(self, value):
        components = urllib.parse.urlparse(value)

        # dont normalise path for http URI's
        if components.scheme and not components.scheme == "http":
            normalized_path = posixpath.normpath(components.path)
            if normalized_path == ".":
                normalized_path = ""

            components = components._replace(path=normalized_path)
        if not components.scheme:
            # For file:// URNs, we need to parse them from a filename.
            components = components._replace(
                netloc="",
                path=urllib.request.pathname2url(value),
                scheme="file")
            self.original_filename = value

        return components 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def __init__(self, header):
        hlines = header.split(b'\n')
        req = hlines[0]
        reqparts = req.split(b" ")
        self.type = reqparts[0]
        self.path = urllib.parse.unquote(reqparts[1].decode('utf-8'))
        self.version = reqparts[2]
        r = self.version.find(b"\r")
        if r:
            self.version = self.version[:r]
        self.headers = {}
        for hline in hlines[1:]:
            if hline[len(hline)-1] == 13: # \r
                hline = hline[:len(hline)-1]
            hset = hline.split(b":", 1)
            self.headers[hset[0].lower()] = hset[1].strip() 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import parse [as 别名]
def parse_table_definition_file(file):
    """
    Read an parse the XML of a table-definition file.
    @return: an ElementTree object for the table definition
    """
    logging.info("Reading table definition from '%s'...", file)
    if not os.path.isfile(file):
        handle_error("File '%s' does not exist.", file)

    try:
        tableGenFile = ElementTree.ElementTree().parse(file)
    except OSError as e:
        handle_error("Could not read result file %s: %s", file, e)
    except ElementTree.ParseError as e:
        handle_error("Table file %s is invalid: %s", file, e)
    if "table" != tableGenFile.tag:
        handle_error(
            "Table file %s is invalid: It's root element is not named 'table'.", file
        )
    return tableGenFile