Python urllib_parse.urlparse方法代码示例

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def save_report(session, repo, name, url, username):
    """ Save the report of issues based on the given URL of the project.
    """
    url_obj = urlparse(url)
    url = url_obj.geturl().replace(url_obj.query, "")
    query = {}
    for k, v in parse_qsl(url_obj.query):
        if k in query:
            if isinstance(query[k], list):
                query[k].append(v)
            else:
                query[k] = [query[k], v]
        else:
            query[k] = v
    reports = repo.reports
    reports[name] = query
    repo.reports = reports
    session.add(repo) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def _idna_encode(self, value):
        parsed = urllib_parse.urlparse(value)
        if parsed.port:
            netloc = (
                idna.encode(parsed.hostname) +
                ":{0}".format(parsed.port).encode("ascii")
            ).decode("ascii")
        else:
            netloc = idna.encode(parsed.hostname).decode("ascii")

        # Note that building a URL in this fashion means it should be
        # semantically indistinguishable from the original but is not
        # guaranteed to be exactly the same.
        return urllib_parse.urlunparse((
            parsed.scheme,
            netloc,
            parsed.path,
            parsed.params,
            parsed.query,
            parsed.fragment
        )) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def _idna_encode(self, value):
        idna = _lazy_import_idna()
        parsed = urllib_parse.urlparse(value)
        if parsed.port:
            netloc = (
                idna.encode(parsed.hostname) +
                ":{}".format(parsed.port).encode("ascii")
            ).decode("ascii")
        else:
            netloc = idna.encode(parsed.hostname).decode("ascii")

        # Note that building a URL in this fashion means it should be
        # semantically indistinguishable from the original but is not
        # guaranteed to be exactly the same.
        return urllib_parse.urlunparse((
            parsed.scheme,
            netloc,
            parsed.path,
            parsed.params,
            parsed.query,
            parsed.fragment
        )) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def data(self):
        """
        TODO: What is the right way to do this?
        """
        if not self.body:
            return self.body
        elif self.body is EMPTY:
            return EMPTY
        elif self.content_type and self.content_type.startswith('application/json'):
            try:
                if isinstance(self.body, six.binary_type):
                    return json.loads(self.body.decode('utf-8'))
                else:
                    return json.loads(self.body)
            except ValueError as e:
                if isinstance(e, JSONDecodeError):
                    # this will only be True for Python3+
                    raise e
                raise JSONDecodeError(str(e))
        elif self.content_type == 'application/x-www-form-urlencoded':
            return dict(urlparse.parse_qsl(self.body))
        else:
            raise NotImplementedError("No parser for content type") 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def validate_deferred_references(schema, context, **kwargs):
    try:
        deferred_references = context['deferred_references']
    except KeyError:
        raise KeyError("`deferred_references` not found in context")

    with ErrorDict() as errors:
        for reference in deferred_references:
            parts = urlparse.urlparse(reference)
            if any((parts.scheme, parts.netloc, parts.path, parts.params, parts.query)):
                errors.add_error(
                    reference,
                    MESSAGES['reference']['unsupported'].format(reference),
                )
                continue
            try:
                jsonpointer.resolve_pointer(schema, parts.fragment)
            except jsonpointer.JsonPointerException:
                errors.add_error(
                    reference,
                    MESSAGES['reference']['undefined'].format(reference),
                ) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def get_media_url(url, result_blacklist=None, patterns=None, generic_patterns=True):
    if patterns is None:
        patterns = []
    scheme = urllib_parse.urlparse(url).scheme
    if result_blacklist is None:
        result_blacklist = []
    elif isinstance(result_blacklist, str):
        result_blacklist = [result_blacklist]

    result_blacklist = list(set(result_blacklist + ['.smil']))  # smil(not playable) contains potential sources, only blacklist when called from here
    net = common.Net()
    headers = {'User-Agent': common.RAND_UA}
    headers.update({'Referer': url})
    response = net.http_GET(url, headers=headers)
    response_headers = response.get_headers(as_dict=True)
    cookie = response_headers.get('Set-Cookie', None)
    if cookie:
        headers.update({'Cookie': cookie})
    html = response.content

    source_list = scrape_sources(html, result_blacklist, scheme, patterns, generic_patterns)
    source = pick_source(source_list)
    return source + append_headers(headers) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def _make_conf(backend_uri):
    parsed_url = urllib_parse.urlparse(backend_uri)
    backend_type = parsed_url.scheme.lower()
    if not backend_type:
        raise ValueError("Unknown backend type for uri: %s" % (backend_type))
    if backend_type in ('file', 'dir'):
        conf = {
            'path': parsed_url.path,
            'connection': backend_uri,
        }
    elif backend_type in ('zookeeper',):
        conf = {
            'path': parsed_url.path,
            'hosts': parsed_url.netloc,
            'connection': backend_uri,
        }
    else:
        conf = {
            'connection': backend_uri,
        }
    return conf 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def _attach_auth_cookies(self):
    auth_url = self.get_argument(_AUTH_URL_QUERY_PARAM, default='')
    if not auth_url:
      raise gen.Return()

    parsed_auth_url = urlparse.urlparse(auth_url)

    try:
      _validate_same_domain(self.request, parsed_auth_url)
      extra_cookies = yield _perform_request_and_extract_cookies(
          parsed_auth_url, self.ca_certs, self._get_http_client())
    except Exception:  # pylint:disable=broad-except
      self.log.exception('Uncaught error when proxying request')
      raise

    self.request.headers.update(extra_cookies) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def handle_state(self, start_url, target_url):
        start_query = parse_qs(urlparse(start_url).query)
        redirect_uri = start_query.get('redirect_uri')

        if getattr(self.backend, 'STATE_PARAMETER', False):
            if start_query.get('state'):
                target_url = url_add_parameters(target_url, {
                    'state': start_query['state']
                })

        if redirect_uri and getattr(self.backend, 'REDIRECT_STATE', False):
            redirect_query = parse_qs(urlparse(redirect_uri).query)
            if redirect_query.get('redirect_state'):
                target_url = url_add_parameters(target_url, {
                    'redirect_state': redirect_query['redirect_state']
                })
        return target_url 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def do_start(self):
        start_url = self.backend.start().url
        # Modify the start URL to make the SAML request consistent
        # from test to test:
        start_url = self.modify_start_url(start_url)
        # If the SAML Identity Provider recognizes the user, we will
        # be redirected back to:
        return_url = self.backend.redirect_uri
        self.install_http_intercepts(start_url, return_url)
        response = requests.get(start_url)
        self.assertTrue(response.url.startswith(return_url))
        self.assertEqual(response.text, 'foobar')
        query_values = dict((k, v[0]) for k, v in
                            parse_qs(urlparse(response.url).query).items())
        self.assertNotIn(' ', query_values['SAMLResponse'])
        self.strategy.set_request_data(query_values, self.backend)
        return self.backend.complete() 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def modify_start_url(self, start_url):
        """
        Given a SAML redirect URL, parse it and change the ID to
        a consistent value, so the request is always identical.
        """
        # Parse the SAML Request URL to get the XML being sent to TestShib
        url_parts = urlparse(start_url)
        query = dict((k, v[0]) for (k, v) in
                     parse_qs(url_parts.query).items())
        xml = OneLogin_Saml2_Utils.decode_base64_and_inflate(
            query['SAMLRequest']
        )
        # Modify the XML:
        xml = xml.decode()
        xml, changed = re.subn(r'ID="[^"]+"', 'ID="TEST_ID"', xml)
        self.assertEqual(changed, 1)
        # Update the URL to use the modified query string:
        query['SAMLRequest'] = OneLogin_Saml2_Utils.deflate_and_base64_encode(
            xml
        )
        url_parts = list(url_parts)
        url_parts[4] = urlencode(query)
        return urlunparse(url_parts) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def get_auth_response(self):
        logger.info('User authentication requires interaction with your '
                    'web browser. Once you enter your credentials and '
                    'give authorization, you will be redirected to '
                    'a url.  Paste that url you were directed to to '
                    'complete the authorization.')

        redirect_info = urlparse(self.redirect_uri)
        redirect_host, redirect_port = get_host_port(redirect_info.netloc)

        if redirect_host in ("127.0.0.1", "localhost") and redirect_info.scheme == "http":
            # Only start a local http server if a port is specified
            if redirect_port:
                return self._get_auth_response_local_server(redirect_port)
            else:
                logger.warning('Using `%s` as redirect URI without a port. '
                               'Specify a port (e.g. `%s:8080`) to allow '
                               'automatic retrieval of authentication code '
                               'instead of having to copy and paste '
                               'the URL your browser is redirected to.',
                               redirect_host, redirect_host)

        logger.info('Paste that url you were directed to in order to '
                    'complete the authorization')
        return self._get_auth_response_interactive() 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def parse_dcs(dcs):
    if dcs is None:
        return None
    elif '//' not in dcs:
        dcs = '//' + dcs

    parsed = urlparse(dcs)
    scheme = parsed.scheme
    port = int(parsed.port) if parsed.port else None

    if scheme == '':
        scheme = ([k for k, v in DCS_DEFAULTS.items() if v['port'] == port] or ['etcd'])[0]
    elif scheme not in DCS_DEFAULTS:
        raise PatroniCtlException('Unknown dcs scheme: {}'.format(scheme))

    default = DCS_DEFAULTS[scheme]
    return yaml.safe_load(default['template'].format(host=parsed.hostname or 'localhost', port=port or default['port'])) 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def filter_img_src(name, value):
    """ Filter in img html tags images coming from a different domain. """
    if name in ("alt", "height", "width", "class", "data-src"):
        return True
    if name == "src":
        parsed = urlparse(value)
        return (not parsed.netloc) or parsed.netloc == urlparse(
            pagure_config["APP_URL"]
        ).netloc
    return False 

# 需要导入模块: from six.moves import urllib_parse [as 别名]
# 或者: from six.moves.urllib_parse import urlparse [as 别名]
def _normalize_url_with_hostname(url):
    # type: (Text) -> Text
    # TODO: replace urlparse by more robust utf-8 handling code
    parsed = urlparse(url.encode("utf-8"))
    hostname = parsed.hostname.decode("utf-8")  # type: ignore
    try:
        ipaddress.ip_address(hostname)
        hostname = socket.gethostbyaddr(hostname)[0].encode("utf-8")
    except ValueError:
        return url
    return parsed._replace(netloc=b"%s:%d" % (hostname, parsed.port)).geturl()