Python URL.join方法代码示例

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]

#.........这里部分代码省略.........

        redirects = 0
        history = []

        # Merge with default headers and transform to CIMultiDict
        headers = self._prepare_headers(headers)
        if auth is None:
            auth = self._default_auth
        # It would be confusing if we support explicit Authorization header
        # with `auth` argument
        if (headers is not None and
                auth is not None and
                hdrs.AUTHORIZATION in headers):
            raise ValueError("Can't combine `Authorization` header with "
                             "`auth` argument")

        skip_headers = set(self._skip_auto_headers)
        if skip_auto_headers is not None:
            for i in skip_auto_headers:
                skip_headers.add(istr(i))

        if proxy is not None:
            proxy = URL(proxy)

        while True:
            url = URL(url).with_fragment(None)

            cookies = self._cookie_jar.filter_cookies(url)

            req = self._request_class(
                method, url, params=params, headers=headers,
                skip_auto_headers=skip_headers, data=data,
                cookies=cookies, encoding=encoding,
                auth=auth, version=version, compress=compress, chunked=chunked,
                expect100=expect100,
                loop=self._loop, response_class=self._response_class,
                proxy=proxy, proxy_auth=proxy_auth, timeout=timeout)

            with Timeout(timeout, loop=self._loop):
                conn = yield from self._connector.connect(req)
            try:
                resp = req.send(conn.writer, conn.reader)
                try:
                    yield from resp.start(conn, read_until_eof)
                except:
                    resp.close()
                    conn.close()
                    raise
            except (aiohttp.HttpProcessingError,
                    aiohttp.ServerDisconnectedError) as exc:
                raise aiohttp.ClientResponseError() from exc
            except OSError as exc:
                raise aiohttp.ClientOSError(*exc.args) from exc

            self._cookie_jar.update_cookies(resp.cookies, resp.url_obj)

            # redirects
            if resp.status in (301, 302, 303, 307) and allow_redirects:
                redirects += 1
                history.append(resp)
                if max_redirects and redirects >= max_redirects:
                    resp.close()
                    break
                else:
                    # TODO: close the connection if BODY is large enough
                    # Redirect with big BODY is forbidden by HTTP protocol
                    # but malformed server may send illegal response.
                    # Small BODIES with text like "Not Found" are still
                    # perfectly fine and should be accepted.
                    yield from resp.release()

                # For 301 and 302, mimic IE behaviour, now changed in RFC.
                # Details: https://github.com/kennethreitz/requests/pull/269
                if (resp.status == 303 and resp.method != hdrs.METH_HEAD) \
                   or (resp.status in (301, 302) and
                       resp.method == hdrs.METH_POST):
                    method = hdrs.METH_GET
                    data = None
                    if headers.get(hdrs.CONTENT_LENGTH):
                        headers.pop(hdrs.CONTENT_LENGTH)

                r_url = URL(resp.headers.get(hdrs.LOCATION) or
                            resp.headers.get(hdrs.URI))

                scheme = r_url.scheme
                if scheme not in ('http', 'https', ''):
                    resp.close()
                    raise ValueError('Can redirect only to http or https')
                elif not scheme:
                    r_url = url.join(r_url)

                url = r_url
                params = None
                yield from resp.release()
                continue

            break

        resp._history = tuple(history)
        return resp

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
class BaseTestServer(ABC):
    def __init__(self, *, scheme=sentinel, loop=None,
                 host='127.0.0.1', port=None, skip_url_asserts=False,
                 **kwargs):
        self._loop = loop
        self.runner = None
        self._root = None
        self.host = host
        self.port = port
        self._closed = False
        self.scheme = scheme
        self.skip_url_asserts = skip_url_asserts

    async def start_server(self, loop=None, **kwargs):
        if self.runner:
            return
        self._loop = loop
        self._ssl = kwargs.pop('ssl', None)
        self.runner = await self._make_runner(**kwargs)
        await self.runner.setup()
        if not self.port:
            self.port = unused_port()
        site = TCPSite(self.runner, host=self.host, port=self.port,
                       ssl_context=self._ssl)
        await site.start()
        if self.scheme is sentinel:
            if self._ssl:
                scheme = 'https'
            else:
                scheme = 'http'
            self.scheme = scheme
        self._root = URL('{}://{}:{}'.format(self.scheme,
                                             self.host,
                                             self.port))

    @abstractmethod  # pragma: no cover
    async def _make_runner(self, **kwargs):
        pass

    def make_url(self, path):
        url = URL(path)
        if not self.skip_url_asserts:
            assert not url.is_absolute()
            return self._root.join(url)
        else:
            return URL(str(self._root) + path)

    @property
    def started(self):
        return self.runner is not None

    @property
    def closed(self):
        return self._closed

    @property
    def handler(self):
        # for backward compatibility
        # web.Server instance
        return self.runner.server

    async def close(self):
        """Close all fixtures created by the test client.

        After that point, the TestClient is no longer usable.

        This is an idempotent function: running close multiple times
        will not have any additional effects.

        close is also run when the object is garbage collected, and on
        exit when used as a context manager.

        """
        if self.started and not self.closed:
            await self.runner.cleanup()
            self._root = None
            self.port = None
            self._closed = True

    def __enter__(self):
        raise TypeError("Use async with instead")

    def __exit__(self, exc_type, exc_value, traceback):
        # __exit__ should exist in pair with __enter__ but never executed
        pass  # pragma: no cover

    async def __aenter__(self):
        await self.start_server(loop=self._loop)
        return self

    async def __aexit__(self, exc_type, exc_value, traceback):
        await self.close()

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
def test_join_from_rfc_3986_abnormal(url, expected):
    # test case from https://tools.ietf.org/html/rfc3986.html#section-5.4.2
    base = URL("http://a/b/c/d;p?q")
    url = URL(url)
    expected = URL(expected)
    assert base.join(url) == expected

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]

#.........这里部分代码省略.........
        if skip_auto_headers is not None:
            for i in skip_auto_headers:
                skip_headers.add(istr(i))

        if proxy is not None:
            proxy = URL(proxy)

        # request timeout
        if timeout is None:
            timeout = self._read_timeout
        if timeout is None:
            timeout = self._connector.conn_timeout
        elif self._connector.conn_timeout is not None:
            timeout = max(timeout, self._connector.conn_timeout)

        # timeout is cumulative for all request operations
        # (request, redirects, responses, data consuming)
        timer = self._time_service.timeout(timeout)

        with timer:
            while True:
                url = URL(url).with_fragment(None)

                cookies = self._cookie_jar.filter_cookies(url)

                req = self._request_class(
                    method, url, params=params, headers=headers,
                    skip_auto_headers=skip_headers, data=data,
                    cookies=cookies, encoding=encoding,
                    auth=auth, version=version, compress=compress,
                    chunked=chunked, expect100=expect100,
                    loop=self._loop, response_class=self._response_class,
                    proxy=proxy, proxy_auth=proxy_auth, timer=timer)

                conn = yield from self._connector.connect(req)
                conn.writer.set_tcp_nodelay(True)
                try:
                    resp = req.send(conn)
                    try:
                        yield from resp.start(conn, read_until_eof)
                    except:
                        resp.close()
                        conn.close()
                        raise
                except ClientError:
                    raise
                except http.HttpProcessingError as exc:
                    raise ClientResponseError(
                        code=exc.code,
                        message=exc.message, headers=exc.headers) from exc
                except OSError as exc:
                    raise ClientOSError(*exc.args) from exc

                self._cookie_jar.update_cookies(resp.cookies, resp.url)

                # redirects
                if resp.status in (301, 302, 303, 307) and allow_redirects:
                    redirects += 1
                    history.append(resp)
                    if max_redirects and redirects >= max_redirects:
                        resp.close()
                        break
                    else:
                        yield from resp.release()

                    # For 301 and 302, mimic IE behaviour, now changed in RFC.
                    # Info: https://github.com/kennethreitz/requests/pull/269
                    if (resp.status == 303 and resp.method != hdrs.METH_HEAD) \
                       or (resp.status in (301, 302) and
                           resp.method == hdrs.METH_POST):
                        method = hdrs.METH_GET
                        data = None
                        if headers.get(hdrs.CONTENT_LENGTH):
                            headers.pop(hdrs.CONTENT_LENGTH)

                    r_url = (resp.headers.get(hdrs.LOCATION) or
                             resp.headers.get(hdrs.URI))
                    if r_url is None:
                        raise RuntimeError("{0.method} {0.url} returns "
                                           "a redirect [{0.status}] status "
                                           "but response lacks a Location "
                                           "or URI HTTP header".format(resp))
                    r_url = URL(r_url)

                    scheme = r_url.scheme
                    if scheme not in ('http', 'https', ''):
                        resp.close()
                        raise ValueError('Can redirect only to http or https')
                    elif not scheme:
                        r_url = url.join(r_url)

                    url = r_url
                    params = None
                    yield from resp.release()
                    continue

                break

        resp._history = tuple(history)
        return resp

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
def test_join_absolute():
    base = URL("http://www.cwi.nl/%7Eguido/Python.html")
    url = URL("//www.python.org/%7Eguido")
    url2 = base.join(url)
    assert str(url2) == "http://www.python.org/~guido"

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
def test_join_non_url():
    base = URL("http://example.com")
    with pytest.raises(TypeError):
        base.join("path/to")

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
def test_join():
    base = URL("http://www.cwi.nl/%7Eguido/Python.html")
    url = URL("FAQ.html")
    url2 = base.join(url)
    assert str(url2) == "http://www.cwi.nl/~guido/FAQ.html"

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]

#.........这里部分代码省略.........
                            await trace.send_request_redirect(
                                method,
                                url,
                                headers,
                                resp
                            )

                        redirects += 1
                        history.append(resp)
                        if max_redirects and redirects >= max_redirects:
                            resp.close()
                            raise TooManyRedirects(
                                history[0].request_info, tuple(history))

                        # For 301 and 302, mimic IE, now changed in RFC
                        # https://github.com/kennethreitz/requests/pull/269
                        if (resp.status == 303 and
                                resp.method != hdrs.METH_HEAD) \
                                or (resp.status in (301, 302) and
                                    resp.method == hdrs.METH_POST):
                            method = hdrs.METH_GET
                            data = None
                            if headers.get(hdrs.CONTENT_LENGTH):
                                headers.pop(hdrs.CONTENT_LENGTH)

                        r_url = (resp.headers.get(hdrs.LOCATION) or
                                 resp.headers.get(hdrs.URI))
                        if r_url is None:
                            # see github.com/aio-libs/aiohttp/issues/2022
                            break
                        else:
                            # reading from correct redirection
                            # response is forbidden
                            resp.release()

                        try:
                            r_url = URL(
                                r_url, encoded=not self._requote_redirect_url)

                        except ValueError:
                            raise InvalidURL(r_url)

                        scheme = r_url.scheme
                        if scheme not in ('http', 'https', ''):
                            resp.close()
                            raise ValueError(
                                'Can redirect only to http or https')
                        elif not scheme:
                            r_url = url.join(r_url)

                        if url.origin() != r_url.origin():
                            auth = None
                            headers.pop(hdrs.AUTHORIZATION, None)

                        url = r_url
                        params = None
                        resp.release()
                        continue

                    break

            # check response status
            if raise_for_status is None:
                raise_for_status = self._raise_for_status
            if raise_for_status:
                resp.raise_for_status()

            # register connection
            if handle is not None:
                if resp.connection is not None:
                    resp.connection.add_callback(handle.cancel)
                else:
                    handle.cancel()

            resp._history = tuple(history)

            for trace in traces:
                await trace.send_request_end(
                    method,
                    url,
                    headers,
                    resp
                )
            return resp

        except BaseException as e:
            # cleanup timer
            tm.close()
            if handle:
                handle.cancel()
                handle = None

            for trace in traces:
                await trace.send_request_exception(
                    method,
                    url,
                    headers,
                    e
                )
            raise

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]
class BaseTestServer(ABC):
    def __init__(self,
                 *,
                 scheme: Union[str, object]=sentinel,
                 loop: Optional[asyncio.AbstractEventLoop]=None,
                 host: str='127.0.0.1',
                 port: Optional[int]=None,
                 skip_url_asserts: bool=False,
                 **kwargs: Any) -> None:
        self._loop = loop
        self.runner = None  # type: Optional[BaseRunner]
        self._root = None  # type: Optional[URL]
        self.host = host
        self.port = port
        self._closed = False
        self.scheme = scheme
        self.skip_url_asserts = skip_url_asserts

    async def start_server(self,
                           loop: Optional[asyncio.AbstractEventLoop]=None,
                           **kwargs: Any) -> None:
        if self.runner:
            return
        self._loop = loop
        self._ssl = kwargs.pop('ssl', None)
        self.runner = await self._make_runner(**kwargs)
        await self.runner.setup()
        if not self.port:
            self.port = 0
        _sock = get_port_socket(self.host, self.port)
        self.host, self.port = _sock.getsockname()[:2]
        site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl)
        await site.start()
        server = site._server
        assert server is not None
        sockets = server.sockets
        assert sockets is not None
        self.port = sockets[0].getsockname()[1]
        if self.scheme is sentinel:
            if self._ssl:
                scheme = 'https'
            else:
                scheme = 'http'
            self.scheme = scheme
        self._root = URL('{}://{}:{}'.format(self.scheme,
                                             self.host,
                                             self.port))

    @abstractmethod  # pragma: no cover
    async def _make_runner(self, **kwargs: Any) -> BaseRunner:
        pass

    def make_url(self, path: str) -> URL:
        assert self._root is not None
        url = URL(path)
        if not self.skip_url_asserts:
            assert not url.is_absolute()
            return self._root.join(url)
        else:
            return URL(str(self._root) + path)

    @property
    def started(self) -> bool:
        return self.runner is not None

    @property
    def closed(self) -> bool:
        return self._closed

    @property
    def handler(self) -> Server:
        # for backward compatibility
        # web.Server instance
        runner = self.runner
        assert runner is not None
        assert runner.server is not None
        return runner.server

    async def close(self) -> None:
        """Close all fixtures created by the test client.

        After that point, the TestClient is no longer usable.

        This is an idempotent function: running close multiple times
        will not have any additional effects.

        close is also run when the object is garbage collected, and on
        exit when used as a context manager.

        """
        if self.started and not self.closed:
            assert self.runner is not None
            await self.runner.cleanup()
            self._root = None
            self.port = None
            self._closed = True

    def __enter__(self) -> None:
        raise TypeError("Use async with instead")

#.........这里部分代码省略.........

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]

#.........这里部分代码省略.........
                        cookies=cookies, auth=auth, version=version,
                        compress=compress, chunked=chunked,
                        expect100=expect100, loop=self._loop,
                        response_class=self._response_class,
                        proxy=proxy, proxy_auth=proxy_auth, timer=timer)

                    # connection timeout
                    try:
                        with CeilTimeout(self._conn_timeout, loop=self._loop):
                            conn = yield from self._connector.connect(req)
                    except asyncio.TimeoutError as exc:
                        raise ServerTimeoutError(
                            'Connection timeout '
                            'to host {0}'.format(url)) from exc

                    conn.writer.set_tcp_nodelay(True)
                    try:
                        resp = req.send(conn)
                        try:
                            yield from resp.start(conn, read_until_eof)
                        except:
                            resp.close()
                            conn.close()
                            raise
                    except ClientError:
                        raise
                    except OSError as exc:
                        raise ClientOSError(*exc.args) from exc

                    self._cookie_jar.update_cookies(resp.cookies, resp.url)

                    # redirects
                    if resp.status in (301, 302, 303, 307) and allow_redirects:
                        redirects += 1
                        history.append(resp)
                        if max_redirects and redirects >= max_redirects:
                            resp.close()
                            break
                        else:
                            resp.release()

                        # For 301 and 302, mimic IE, now changed in RFC
                        # https://github.com/kennethreitz/requests/pull/269
                        if (resp.status == 303 and
                                resp.method != hdrs.METH_HEAD) \
                                or (resp.status in (301, 302) and
                                    resp.method == hdrs.METH_POST):
                            method = hdrs.METH_GET
                            data = None
                            if headers.get(hdrs.CONTENT_LENGTH):
                                headers.pop(hdrs.CONTENT_LENGTH)

                        r_url = (resp.headers.get(hdrs.LOCATION) or
                                 resp.headers.get(hdrs.URI))
                        if r_url is None:
                            raise RuntimeError(
                                "{0.method} {0.url} returns "
                                "a redirect [{0.status}] status "
                                "but response lacks a Location "
                                "or URI HTTP header".format(resp))
                        r_url = URL(
                            r_url, encoded=not self.requote_redirect_url)

                        scheme = r_url.scheme
                        if scheme not in ('http', 'https', ''):
                            resp.close()
                            raise ValueError(
                                'Can redirect only to http or https')
                        elif not scheme:
                            r_url = url.join(r_url)

                        url = r_url
                        params = None
                        resp.release()
                        continue

                    break

            # check response status
            if self._raise_for_status:
                resp.raise_for_status()

            # register connection
            if handle is not None:
                if resp.connection is not None:
                    resp.connection.add_callback(handle.cancel)
                else:
                    handle.cancel()

            resp._history = tuple(history)
            return resp

        except:
            # cleanup timer
            tm.close()
            if handle:
                handle.cancel()
                handle = None

            raise

# 需要导入模块: from yarl import URL [as 别名]
# 或者: from yarl.URL import join [as 别名]

#.........这里部分代码省略.........
                    # redirects
                    if resp.status in (
                            301, 302, 303, 307, 308) and allow_redirects:

                        for trace in traces:
                            await trace.send_request_redirect(
                                method,
                                url,
                                headers,
                                resp
                            )

                        redirects += 1
                        history.append(resp)
                        if max_redirects and redirects >= max_redirects:
                            resp.close()
                            break
                        else:
                            resp.release()

                        # For 301 and 302, mimic IE, now changed in RFC
                        # https://github.com/kennethreitz/requests/pull/269
                        if (resp.status == 303 and
                                resp.method != hdrs.METH_HEAD) \
                                or (resp.status in (301, 302) and
                                    resp.method == hdrs.METH_POST):
                            method = hdrs.METH_GET
                            data = None
                            if headers.get(hdrs.CONTENT_LENGTH):
                                headers.pop(hdrs.CONTENT_LENGTH)

                        r_url = (resp.headers.get(hdrs.LOCATION) or
                                 resp.headers.get(hdrs.URI))
                        if r_url is None:
                            # see github.com/aio-libs/aiohttp/issues/2022
                            break

                        try:
                            r_url = URL(
                                r_url, encoded=not self.requote_redirect_url)

                        except ValueError:
                            raise InvalidURL(r_url)

                        scheme = r_url.scheme
                        if scheme not in ('http', 'https', ''):
                            resp.close()
                            raise ValueError(
                                'Can redirect only to http or https')
                        elif not scheme:
                            r_url = url.join(r_url)

                        if url.origin() != r_url.origin():
                            auth = None
                            headers.pop(hdrs.AUTHORIZATION, None)

                        url = r_url
                        params = None
                        resp.release()
                        continue

                    break

            # check response status
            if self._raise_for_status:
                resp.raise_for_status()

            # register connection
            if handle is not None:
                if resp.connection is not None:
                    resp.connection.add_callback(handle.cancel)
                else:
                    handle.cancel()

            resp._history = tuple(history)

            for trace in traces:
                await trace.send_request_end(
                    method,
                    url,
                    headers,
                    resp
                )
            return resp

        except Exception as e:
            # cleanup timer
            tm.close()
            if handle:
                handle.cancel()
                handle = None

            for trace in traces:
                await trace.send_request_exception(
                    method,
                    url,
                    headers,
                    e
                )
            raise