Retries for connection failures

docs/advanced.md

@@ -471,3 +471,56 @@ If you do need to make HTTPS connections to a local server, for example to test
 >>> r
 Response <200 OK>
 ```
+
+## Retries
+
+Communicating with a peer over a network is by essence subject to errors. HTTPX provides built-in retry functionality to increase the resilience to connection issues.
+
+Retries are disabled by default. When retries are enabled, HTTPX will retry sending the request up to the specified number of times. This behavior is restricted to **connection failures only**, i.e.:
+
+* Failures to establish or acquire a connection (`ConnectTimeout`, `PoolTimeout`).
+* Failures to keep the connection open (`NetworkError`).
+
+!!! important
+    HTTPX will **NOT** retry on failures that aren't related to establishing or maintaining connections. This includes in particular:
+
+    * Errors related to data transfer, such as `ReadTimeout` or `ProtocolError`.
+    * HTTP error responses (4xx, 5xx), such as `429 Too Many Requests` or `503 Service Unavailable`.
+
+If HTTPX could not get a response after the specified number of retries, a `TooManyRetries` exception is raised.
+
+The delay between each retry is increased exponentially to prevent overloading the requested host.
+
+### Enabling retries
+
+You can enable retries for a given request:
+
+```python
+# Using the top-level API:
+response = httpx.get("https://example.org", retries=3)
+
+# Using a client instance:
+with httpx.Client() as client:
+    response = client.get("https://example.org", retries=3)
+```
+
+Or enable them on a client instance, which results in the given `retries` being used as a default for requests made with this client:
+
+```python
+# Retry at most 3 times on connection failures everywhere.
+httpx.Client(retries=3)
+```
+
+### Fine-tuning the retries configuration
+
+When enabling retries, the `retries` argument can also be an `httpx.Retries()` instance. It accepts the following arguments:
+
+* An integer, given as a required positional argument, representing the maximum number of connection failures to retry on.
+* `backoff_factor` (optional), which defines the increase rate of the time to wait between retries. By default this is `0.2`, which corresponds to issuing a new request after `(0s, 0.2s, 0.4s, 0.8s, ...)`. (Note that most connection failures are immediately resolved by retrying, so HTTPX will always issue the initial retry right away.)
+
+```python
+# Retry at most 5 times on connection failures,
+# and issue new requests after `(0s, 0.5s, 1s, 2s, 4s, ...)`.
+retries = httpx.Retries(5, backoff_factor=0.5)
+client = httpx.Client(retries=retries)
+```

httpx/__init__.py

@@ -2,7 +2,7 @@
 from .api import delete, get, head, options, patch, post, put, request, stream
 from .auth import Auth, BasicAuth, DigestAuth
 from .client import AsyncClient, Client
-from .config import PoolLimits, Proxy, Timeout
+from .config import PoolLimits, Proxy, Retries, Timeout
 from .dispatch.asgi import ASGIDispatch
 from .dispatch.wsgi import WSGIDispatch
 from .exceptions import (
@@ -12,6 +12,7 @@
     DecodingError,
     HTTPError,
     InvalidURL,
+    NetworkError,
     NotRedirectResponse,
     PoolTimeout,
     ProtocolError,
@@ -25,6 +26,7 @@
     StreamConsumed,
     TimeoutException,
     TooManyRedirects,
+    TooManyRetries,
     WriteTimeout,
 )
 from .models import URL, Cookies, Headers, QueryParams, Request, Response
@@ -54,12 +56,15 @@
     "PoolLimits",
     "Proxy",
     "Timeout",
+    "Retries",
+    "TooManyRetries",
     "ConnectTimeout",
     "CookieConflict",
     "ConnectionClosed",
     "DecodingError",
     "HTTPError",
     "InvalidURL",
+    "NetworkError",
     "NotRedirectResponse",
     "PoolTimeout",
     "ProtocolError",

httpx/api.py

@@ -2,7 +2,14 @@
 
 from .auth import AuthTypes
 from .client import Client, StreamContextManager
-from .config import DEFAULT_TIMEOUT_CONFIG, CertTypes, TimeoutTypes, VerifyTypes
+from .config import (
+    DEFAULT_RETRIES_CONFIG,
+    DEFAULT_TIMEOUT_CONFIG,
+    CertTypes,
+    RetriesTypes,
+    TimeoutTypes,
+    VerifyTypes,
+)
 from .models import (
     CookieTypes,
     HeaderTypes,
@@ -26,6 +33,7 @@ def request(
     headers: HeaderTypes = None,
     cookies: CookieTypes = None,
     auth: AuthTypes = None,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     allow_redirects: bool = True,
     verify: VerifyTypes = True,
@@ -54,6 +62,8 @@ def request(
     request.
     * **auth** - *(optional)* An authentication class to use when sending the
     request.
+    * **retries** - *(optional)* The maximum number of connection failures to
+    retry on.
     * **timeout** - *(optional)* The timeout configuration to use when sending
     the request.
     * **allow_redirects** - *(optional)* Enables or disables HTTP redirects.
@@ -81,7 +91,7 @@ def request(
     ```
     """
     with Client(
-        cert=cert, verify=verify, timeout=timeout, trust_env=trust_env,
+        cert=cert, verify=verify, retries=retries, timeout=timeout, trust_env=trust_env,
     ) as client:
         return client.request(
             method=method,
@@ -108,13 +118,14 @@ def stream(
     headers: HeaderTypes = None,
     cookies: CookieTypes = None,
     auth: AuthTypes = None,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     allow_redirects: bool = True,
     verify: VerifyTypes = True,
     cert: CertTypes = None,
     trust_env: bool = True,
 ) -> StreamContextManager:
-    client = Client(cert=cert, verify=verify, trust_env=trust_env)
+    client = Client(cert=cert, verify=verify, retries=retries, trust_env=trust_env)
     request = Request(
         method=method,
         url=url,
@@ -145,6 +156,7 @@ def get(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -166,6 +178,7 @@ def get(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -181,6 +194,7 @@ def options(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -202,6 +216,7 @@ def options(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -217,6 +232,7 @@ def head(
     allow_redirects: bool = False,  # Note: Differs to usual default.
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -240,6 +256,7 @@ def head(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -258,6 +275,7 @@ def post(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -279,6 +297,7 @@ def post(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -297,6 +316,7 @@ def put(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -318,6 +338,7 @@ def put(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -336,6 +357,7 @@ def patch(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -357,6 +379,7 @@ def patch(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )
@@ -372,6 +395,7 @@ def delete(
     allow_redirects: bool = True,
     cert: CertTypes = None,
     verify: VerifyTypes = True,
+    retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
     timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
     trust_env: bool = True,
 ) -> Response:
@@ -393,6 +417,7 @@ def delete(
         allow_redirects=allow_redirects,
         cert=cert,
         verify=verify,
+        retries=retries,
         timeout=timeout,
         trust_env=trust_env,
     )

httpx/backends/asyncio.py

@@ -225,6 +225,9 @@ async def open_uds_stream(
 
         return SocketStream(stream_reader=stream_reader, stream_writer=stream_writer)
 
+    async def sleep(self, seconds: float) -> None:
+        await asyncio.sleep(seconds)
+
     def time(self) -> float:
         loop = asyncio.get_event_loop()
         return loop.time()

httpx/backends/auto.py

@@ -41,6 +41,9 @@ async def open_uds_stream(
     ) -> BaseSocketStream:
         return await self.backend.open_uds_stream(path, hostname, ssl_context, timeout)
 
+    async def sleep(self, seconds: float) -> None:
+        await self.backend.sleep(seconds)
+
     def time(self) -> float:
         return self.backend.time()
 

httpx/backends/base.py

@@ -111,6 +111,9 @@ async def open_uds_stream(
     ) -> BaseSocketStream:
         raise NotImplementedError()  # pragma: no cover
 
+    async def sleep(self, seconds: float) -> None:
+        raise NotImplementedError()  # pragma: no cover
+
     def time(self) -> float:
         raise NotImplementedError()  # pragma: no cover
 

httpx/backends/sync.py

@@ -0,0 +1,6 @@
+import time
+
+
+class SyncBackend:
+    def sleep(self, seconds: float) -> None:
+        time.sleep(seconds)

httpx/backends/trio.py

@@ -131,6 +131,9 @@ async def open_uds_stream(
 
         raise ConnectTimeout()
 
+    async def sleep(self, seconds: float) -> None:
+        await trio.sleep(seconds)
+
     def time(self) -> float:
         return trio.current_time()