From 439c732f5bf29a83779dce6da4a55e7efd433cf8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 21 Nov 2017 19:16:11 +0200 Subject: [PATCH] Drop await aiohttp.request() syntax for sake of context manager (#2541) * Get rid of more yield froms * Fix #2540: Drop await aiohttp.request() syntax for sake of context manager * Fix typo * Work on dropping Python 3.4 functionality * Fix typo * Fix flake8 * Fix spelling * Code cleanup --- CHANGES/2540.feature | 2 + aiohttp/client.py | 190 ++++++++++++----------- aiohttp/helpers.py | 79 ---------- aiohttp/test_utils.py | 10 +- docs/client_reference.rst | 46 +++--- docs/multipart.rst | 4 +- tests/test_client_functional.py | 29 +++- tests/test_client_functional_oldstyle.py | 42 ++--- tests/test_client_request.py | 2 +- tests/test_client_session.py | 46 +++--- tests/test_client_ws.py | 6 +- tests/test_connector.py | 8 +- tests/test_helpers.py | 25 --- tests/test_proxy_functional.py | 2 +- tests/test_test_utils.py | 2 +- 15 files changed, 201 insertions(+), 292 deletions(-) create mode 100644 CHANGES/2540.feature diff --git a/CHANGES/2540.feature b/CHANGES/2540.feature new file mode 100644 index 00000000000..7712be4412e --- /dev/null +++ b/CHANGES/2540.feature @@ -0,0 +1,2 @@ +Drop `resp = await aiohttp.request(...)` syntax for sake of `async +with aiohttp.request(...) as resp:`. diff --git a/aiohttp/client.py b/aiohttp/client.py index 6df3de3819a..0288bf1bafa 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -8,6 +8,7 @@ import sys import traceback import warnings +from collections.abc import Coroutine from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr from yarl import URL @@ -23,8 +24,7 @@ from .connector import * # noqa from .connector import TCPConnector from .cookiejar import CookieJar -from .helpers import (CeilTimeout, TimeoutHandle, _BaseCoroMixin, - deprecated_noop, proxies_from_env, sentinel, +from .helpers import (CeilTimeout, TimeoutHandle, proxies_from_env, sentinel, strip_auth_from_url) from .http import WS_KEY, WebSocketReader, WebSocketWriter from .http_websocket import WSHandshakeError, ws_ext_gen, ws_ext_parse @@ -145,29 +145,28 @@ def request(self, method, url, **kwargs): """Perform HTTP request.""" return _RequestContextManager(self._request(method, url, **kwargs)) - @asyncio.coroutine - def _request(self, method, url, *, - params=None, - data=None, - json=None, - headers=None, - skip_auto_headers=None, - auth=None, - allow_redirects=True, - max_redirects=10, - encoding=None, - compress=None, - chunked=None, - expect100=False, - read_until_eof=True, - proxy=None, - proxy_auth=None, - timeout=sentinel, - verify_ssl=None, - fingerprint=None, - ssl_context=None, - proxy_headers=None, - trace_request_ctx=None): + async def _request(self, method, url, *, + params=None, + data=None, + json=None, + headers=None, + skip_auto_headers=None, + auth=None, + allow_redirects=True, + max_redirects=10, + encoding=None, + compress=None, + chunked=None, + expect100=False, + read_until_eof=True, + proxy=None, + proxy_auth=None, + timeout=sentinel, + verify_ssl=None, + fingerprint=None, + ssl_context=None, + proxy_headers=None, + trace_request_ctx=None): # NOTE: timeout clamps existing connect and read timeouts. We cannot # set the default to None because we need to detect if the user wants @@ -234,7 +233,7 @@ def _request(self, method, url, *, ] for trace in traces: - yield from trace.send_request_start( + await trace.send_request_start( method, url, headers @@ -288,7 +287,7 @@ def _request(self, method, url, *, # connection timeout try: with CeilTimeout(self._conn_timeout, loop=self._loop): - conn = yield from self._connector.connect( + conn = await self._connector.connect( req, traces=traces ) @@ -301,7 +300,7 @@ def _request(self, method, url, *, try: resp = req.send(conn) try: - yield from resp.start(conn, read_until_eof) + await resp.start(conn, read_until_eof) except Exception: resp.close() conn.close() @@ -318,7 +317,7 @@ def _request(self, method, url, *, 301, 302, 303, 307, 308) and allow_redirects: for trace in traces: - yield from trace.send_request_redirect( + await trace.send_request_redirect( method, url, headers, @@ -390,7 +389,7 @@ def _request(self, method, url, *, resp._history = tuple(history) for trace in traces: - yield from trace.send_request_end( + await trace.send_request_end( method, url, headers, @@ -406,7 +405,7 @@ def _request(self, method, url, *, handle = None for trace in traces: - yield from trace.send_request_exception( + await trace.send_request_exception( method, url, headers, @@ -451,24 +450,23 @@ def ws_connect(self, url, *, proxy_headers=proxy_headers, compress=compress)) - @asyncio.coroutine - def _ws_connect(self, url, *, - protocols=(), - timeout=10.0, - receive_timeout=None, - autoclose=True, - autoping=True, - heartbeat=None, - auth=None, - origin=None, - headers=None, - proxy=None, - proxy_auth=None, - verify_ssl=None, - fingerprint=None, - ssl_context=None, - proxy_headers=None, - compress=0): + async def _ws_connect(self, url, *, + protocols=(), + timeout=10.0, + receive_timeout=None, + autoclose=True, + autoping=True, + heartbeat=None, + auth=None, + origin=None, + headers=None, + proxy=None, + proxy_auth=None, + verify_ssl=None, + fingerprint=None, + ssl_context=None, + proxy_headers=None, + compress=0): if headers is None: headers = CIMultiDict() @@ -495,15 +493,15 @@ def _ws_connect(self, url, *, headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr # send request - resp = yield from self.get(url, headers=headers, - read_until_eof=False, - auth=auth, - proxy=proxy, - proxy_auth=proxy_auth, - verify_ssl=verify_ssl, - fingerprint=fingerprint, - ssl_context=ssl_context, - proxy_headers=proxy_headers) + resp = await self.get(url, headers=headers, + read_until_eof=False, + auth=auth, + proxy=proxy, + proxy_auth=proxy_auth, + verify_ssl=verify_ssl, + fingerprint=fingerprint, + ssl_context=ssl_context, + proxy_headers=proxy_headers) try: # check handshake @@ -663,7 +661,7 @@ def delete(self, url, **kwargs): self._request(hdrs.METH_DELETE, url, **kwargs)) - def close(self): + async def close(self): """Close underlying connector. Release all acquired resources. @@ -673,8 +671,6 @@ def close(self): self._connector.close() self._connector = None - return deprecated_noop('ClientSession.close() is a coroutine') - @property def closed(self): """Is client session closed. @@ -717,32 +713,45 @@ def __exit__(self, exc_type, exc_val, exc_tb): # __exit__ should exist in pair with __enter__ but never executed pass # pragma: no cover - @asyncio.coroutine - def __aenter__(self): + async def __aenter__(self): return self - @asyncio.coroutine - def __aexit__(self, exc_type, exc_val, exc_tb): - yield from self.close() + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() -class _BaseRequestContextManager(_BaseCoroMixin): +class _BaseRequestContextManager(Coroutine): - __slots__ = ('_resp',) + __slots__ = ('_coro', '_resp') def __init__(self, coro): - super().__init__(coro) self._coro = coro - @asyncio.coroutine - def __aenter__(self): - self._resp = yield from self._coro + async def __aenter__(self): + self._resp = await self._coro return self._resp + # @asyncio.coroutine + # def __iter__(self): + # ret = yield from self._coro.__await__() + # return ret + + def send(self, arg): + return self._coro.send(arg) + + def throw(self, arg): + return self._coro.throw(arg) + + def close(self): + return self._coro.close() + + def __await__(self): + ret = self._coro.__await__() + return ret + class _RequestContextManager(_BaseRequestContextManager): - @asyncio.coroutine - def __aexit__(self, exc_type, exc, tb): + async def __aexit__(self, exc_type, exc, tb): # We're basing behavior on the exception as it can be caused by # user code unrelated to the status of the connection. If you # would like to close a connection you must do that @@ -752,33 +761,26 @@ def __aexit__(self, exc_type, exc, tb): class _WSRequestContextManager(_BaseRequestContextManager): - @asyncio.coroutine - def __aexit__(self, exc_type, exc, tb): - yield from self._resp.close() + async def __aexit__(self, exc_type, exc, tb): + await self._resp.close() -class _SessionRequestContextManager(_RequestContextManager): +class _SessionRequestContextManager: - __slots__ = _RequestContextManager.__slots__ + ('_session', ) + __slots__ = ('_coro', '_resp', '_session') def __init__(self, coro, session): - super().__init__(coro) + self._coro = coro + self._resp = None self._session = session - @asyncio.coroutine - def __iter__(self): - try: - return (yield from self._coro) - except Exception: - yield from self._session.close() - raise + async def __aenter__(self): + self._resp = await self._coro + return self._resp - def __await__(self): - try: - return (yield from self._coro) - except Exception: - yield from self._session.close() - raise + async def __aexit__(self, exc_type, exc_val, exc_tb): + self._resp.close() + await self._session.close() def request(method, url, *, @@ -859,4 +861,4 @@ def request(method, url, *, read_until_eof=read_until_eof, proxy=proxy, proxy_auth=proxy_auth,), - session=session) + session) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index a1521ff1447..5ead32ee1c3 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -10,10 +10,8 @@ import os import re import time -import warnings import weakref from collections import namedtuple -from collections.abc import Coroutine from contextlib import suppress from math import ceil from pathlib import Path @@ -41,79 +39,6 @@ TOKEN = CHAR ^ CTL ^ SEPARATORS -class _BaseCoroMixin(Coroutine): - - __slots__ = ('_coro',) - - def __init__(self, coro): - self._coro = coro - - def send(self, arg): - return self._coro.send(arg) - - def throw(self, arg): - return self._coro.throw(arg) - - def close(self): - return self._coro.close() - - @property - def gi_frame(self): - return self._coro.gi_frame - - @property - def gi_running(self): - return self._coro.gi_running - - @property - def gi_code(self): - return self._coro.gi_code - - def __next__(self): - return self.send(None) - - @asyncio.coroutine - def __iter__(self): - ret = yield from self._coro - return ret - - def __await__(self): - ret = yield from self._coro - return ret - - -class _CoroGuard(_BaseCoroMixin): - """Only to be used with func:`deprecated_noop`. - - Otherwise the stack information in the raised warning doesn't line up with - the user's code anymore. - """ - __slots__ = ('_msg', '_awaited') - - def __init__(self, coro, msg): - super().__init__(coro) - self._msg = msg - self._awaited = False - - def send(self, arg): - self._awaited = True - return self._coro.send(arg) - - @asyncio.coroutine - def __iter__(self): - self._awaited = True - return super().__iter__() - - def __await__(self): - self._awaited = True - return super().__await__() - - def __del__(self): - self._coro = None - if not self._awaited: - warnings.warn(self._msg, DeprecationWarning, stacklevel=2) - - coroutines = asyncio.coroutines old_debug = coroutines._DEBUG coroutines._DEBUG = False @@ -124,10 +49,6 @@ def noop(*args, **kwargs): return -def deprecated_noop(message): - return _CoroGuard(noop(), message) - - coroutines._DEBUG = old_debug diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index c8ab9e6aadf..bbf455ba8df 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -220,8 +220,7 @@ def session(self): def make_url(self, path): return self._server.make_url(path) - @asyncio.coroutine - def request(self, method, path, *args, **kwargs): + async def request(self, method, path, *args, **kwargs): """Routes a request to tested http server. The interface is identical to asyncio.ClientSession.request, @@ -229,7 +228,7 @@ def request(self, method, path, *args, **kwargs): test server. """ - resp = yield from self._session.request( + resp = await self._session.request( method, self.make_url(path), *args, **kwargs ) # save it to close later @@ -288,9 +287,8 @@ def ws_connect(self, path, *args, **kwargs): self._ws_connect(path, *args, **kwargs) ) - @asyncio.coroutine - def _ws_connect(self, path, *args, **kwargs): - ws = yield from self._session.ws_connect( + async def _ws_connect(self, path, *args, **kwargs): + ws = await self._session.ws_connect( self.make_url(path), *args, **kwargs) self._websockets.append(ws) return ws diff --git a/docs/client_reference.rst b/docs/client_reference.rst index ecebfd5b00d..a91c9241424 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -310,7 +310,7 @@ The client session supports the context manager protocol for self closing. Perform a ``GET`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. :param url: Request URL, :class:`str` or :class:`~yarl.URL` @@ -328,7 +328,7 @@ The client session supports the context manager protocol for self closing. Perform a ``POST`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. @@ -347,7 +347,7 @@ The client session supports the context manager protocol for self closing. Perform a ``PUT`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. @@ -366,7 +366,7 @@ The client session supports the context manager protocol for self closing. Perform a ``DELETE`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. :param url: Request URL, :class:`str` or :class:`~yarl.URL` @@ -381,7 +381,7 @@ The client session supports the context manager protocol for self closing. Perform a ``HEAD`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. :param url: Request URL, :class:`str` or :class:`~yarl.URL` @@ -399,7 +399,7 @@ The client session supports the context manager protocol for self closing. Perform an ``OPTIONS`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. @@ -418,7 +418,7 @@ The client session supports the context manager protocol for self closing. Perform a ``PATCH`` request. In order to modify inner - :meth:`request` + :meth:`request` parameters, provide `kwargs`. :param url: Request URL, :class:`str` or :class:`~yarl.URL` @@ -541,18 +541,20 @@ keepaliving, cookies and complex connection stuff like properly configured SSL certification chaining. -.. coroutinefunction:: request(method, url, *, params=None, data=None, \ - json=None,\ - headers=None, cookies=None, auth=None, \ - allow_redirects=True, max_redirects=10, \ - encoding='utf-8', \ - version=HttpVersion(major=1, minor=1), \ - compress=None, chunked=None, expect100=False, \ - connector=None, loop=None,\ - read_until_eof=True) +.. cofunction:: request(method, url, *, params=None, data=None, \ + json=None,\ + headers=None, cookies=None, auth=None, \ + allow_redirects=True, max_redirects=10, \ + encoding='utf-8', \ + version=HttpVersion(major=1, minor=1), \ + compress=None, chunked=None, expect100=False, \ + connector=None, loop=None,\ + read_until_eof=True) - Perform an asynchronous HTTP request. Return a response object - (:class:`ClientResponse` or derived from). + :async-with: + + Asynchronous context manager for performing an asynchronous HTTP + request. Returns a :class:`ClientResponse` response object. :param str method: HTTP method @@ -715,7 +717,7 @@ BaseConnector The call may be paused if :attr:`limit` is exhausted until used connections returns to pool. - :param aiohttp.client.ClientRequest request: request object + :param aiohttp.ClientRequest request: request object which is connection initiator. @@ -1533,11 +1535,13 @@ All exceptions are available as members of *aiohttp* module. .. attribute:: value - A :class:`str` instance. Value of Content-Disposition header itself, e.g. ``attachment``. + A :class:`str` instance. Value of Content-Disposition header + itself, e.g. ``attachment``. .. attribute:: filename - A :class:`str` instance. Content filename extracted from parameters. May be ``None``. + A :class:`str` instance. Content filename extracted from + parameters. May be ``None``. .. attribute:: parameters diff --git a/docs/multipart.rst b/docs/multipart.rst index a9d1559a681..bebdb15159c 100644 --- a/docs/multipart.rst +++ b/docs/multipart.rst @@ -175,9 +175,9 @@ and form urlencoded data, so you don't have to encode it every time manually:: mpwriter.append_form([('key', 'value')]) When it's done, to make a request just pass a root :class:`MultipartWriter` -instance as :func:`aiohttp.client.request` `data` argument:: +instance as :meth:`aiohttp.ClientSession.request` ``data`` argument:: - await aiohttp.post('http://example.com', data=mpwriter) + await session.post('http://example.com', data=mpwriter) Behind the scenes :meth:`MultipartWriter.serialize` will yield chunks of every part and if body part has `Content-Encoding` or `Content-Transfer-Encoding` diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index ad1f8fa3426..530bb06e488 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -350,7 +350,7 @@ async def handler(request): assert "{}".format(task).startswith("