Skip to content

Commit

Permalink
Drop DEFAULT_RETRIES_CONFIG for None, add disabling retries per-request
Browse files Browse the repository at this point in the history
  • Loading branch information
florimondmanca committed Jan 22, 2020
1 parent a3e6917 commit d3609c3
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 28 deletions.
6 changes: 6 additions & 0 deletions docs/advanced.md
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,12 @@ Or enable them on a client instance, which results in the given `retries` being
httpx.Client(retries=3)
```

When using a client with retries enabled, you can still explicitly disable retries for a given request:

```python
response = client.get("https://example.org", retries=None)
```

### Fine-tuning the retries configuration

When enabling retries, the `retries` argument can also be an `httpx.Retries()` instance. It accepts the following arguments:
Expand Down
19 changes: 9 additions & 10 deletions httpx/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from .auth import AuthTypes
from .client import Client, StreamContextManager
from .config import (
DEFAULT_RETRIES_CONFIG,
DEFAULT_TIMEOUT_CONFIG,
CertTypes,
RetriesTypes,
Expand Down Expand Up @@ -33,7 +32,7 @@ def request(
headers: HeaderTypes = None,
cookies: CookieTypes = None,
auth: AuthTypes = None,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
allow_redirects: bool = True,
verify: VerifyTypes = True,
Expand Down Expand Up @@ -118,7 +117,7 @@ def stream(
headers: HeaderTypes = None,
cookies: CookieTypes = None,
auth: AuthTypes = None,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
allow_redirects: bool = True,
verify: VerifyTypes = True,
Expand Down Expand Up @@ -156,7 +155,7 @@ def get(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -194,7 +193,7 @@ def options(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -232,7 +231,7 @@ def head(
allow_redirects: bool = False, # Note: Differs to usual default.
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -275,7 +274,7 @@ def post(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -316,7 +315,7 @@ def put(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -357,7 +356,7 @@ def patch(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down Expand Up @@ -395,7 +394,7 @@ def delete(
allow_redirects: bool = True,
cert: CertTypes = None,
verify: VerifyTypes = True,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
trust_env: bool = True,
) -> Response:
Expand Down
37 changes: 24 additions & 13 deletions httpx/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from .config import (
DEFAULT_MAX_REDIRECTS,
DEFAULT_POOL_LIMITS,
DEFAULT_RETRIES_CONFIG,
DEFAULT_TIMEOUT_CONFIG,
UNSET,
CertTypes,
Expand Down Expand Up @@ -68,8 +67,8 @@ def __init__(
params: QueryParamTypes = None,
headers: HeaderTypes = None,
cookies: CookieTypes = None,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
max_redirects: int = DEFAULT_MAX_REDIRECTS,
base_url: URLTypes = None,
trust_env: bool = True,
Expand All @@ -86,8 +85,8 @@ def __init__(
self._params = QueryParams(params)
self._headers = Headers(headers)
self._cookies = Cookies(cookies)
self.timeout = Timeout(timeout)
self.retries = Retries(retries)
self.timeout = Timeout(timeout)
self.max_redirects = max_redirects
self.trust_env = trust_env
self.netrc = NetRCInfo()
Expand Down Expand Up @@ -453,8 +452,8 @@ def __init__(
verify: VerifyTypes = True,
cert: CertTypes = None,
proxies: ProxiesTypes = None,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
max_redirects: int = DEFAULT_MAX_REDIRECTS,
base_url: URLTypes = None,
Expand All @@ -467,8 +466,8 @@ def __init__(
params=params,
headers=headers,
cookies=cookies,
timeout=timeout,
retries=retries,
timeout=timeout,
max_redirects=max_redirects,
base_url=base_url,
trust_env=trust_env,
Expand Down Expand Up @@ -633,6 +632,11 @@ def send_handling_retries(
timeout: Timeout,
allow_redirects: bool = True,
) -> Response:
if not retries.limit:
return self.send_handling_redirects(
request, auth=auth, timeout=timeout, allow_redirects=allow_redirects
)

backend = self.backend
retries_left = retries.limit
delays = retries.get_delays()
Expand All @@ -646,9 +650,10 @@ def send_handling_retries(
allow_redirects=allow_redirects,
)
except HTTPError as exc:
if not retries.limit or not retries.should_retry_on_exception(exc):
# We shouldn't retry at all in these cases, so let's re-raise
# immediately to avoid polluting logs or the exception stack.
if not retries.should_retry_on_exception(exc):
# Even if we have retries left, we're told to not even consider
# retrying in this case. So let's re-raise immediately to avoid
# polluting logs or the exception stack.
raise

logger.debug(f"HTTP Request failed: {exc!r}")
Expand Down Expand Up @@ -1022,8 +1027,8 @@ def __init__(
cert: CertTypes = None,
http2: bool = False,
proxies: ProxiesTypes = None,
retries: RetriesTypes = None,
timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG,
retries: RetriesTypes = DEFAULT_RETRIES_CONFIG,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
max_redirects: int = DEFAULT_MAX_REDIRECTS,
base_url: URLTypes = None,
Expand All @@ -1038,8 +1043,8 @@ def __init__(
params=params,
headers=headers,
cookies=cookies,
timeout=timeout,
retries=retries,
timeout=timeout,
max_redirects=max_redirects,
base_url=base_url,
trust_env=trust_env,
Expand Down Expand Up @@ -1224,6 +1229,11 @@ async def send_handling_retries(
timeout: Timeout,
allow_redirects: bool = True,
) -> Response:
if not retries.limit:
return await self.send_handling_redirects(
request, auth=auth, timeout=timeout, allow_redirects=allow_redirects
)

backend = lookup_backend()

retries_left = retries.limit
Expand All @@ -1238,9 +1248,10 @@ async def send_handling_retries(
allow_redirects=allow_redirects,
)
except HTTPError as exc:
if not retries.limit or not retries.should_retry_on_exception(exc):
# We shouldn't retry at all in these cases, so let's re-raise
# immediately to avoid polluting logs or the exception stack.
if not retries.should_retry_on_exception(exc):
# Even if we have retries left, we're told to not even consider
# retrying in this case. So let's re-raise immediately to avoid
# polluting logs or the exception stack.
raise

logger.debug(f"HTTP Request failed: {exc!r}")
Expand Down
11 changes: 7 additions & 4 deletions httpx/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
ProxiesTypes = typing.Union[
URLTypes, "Proxy", typing.Dict[URLTypes, typing.Union[URLTypes, "Proxy"]]
]
RetriesTypes = typing.Union[int, "Retries"]
RetriesTypes = typing.Union[None, int, "Retries"]


DEFAULT_CIPHERS = ":".join(
Expand Down Expand Up @@ -372,8 +372,12 @@ class Retries:
("HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE")
)

def __init__(self, retries: RetriesTypes, *, backoff_factor: float = None) -> None:
if isinstance(retries, int):
def __init__(
self, retries: RetriesTypes = None, *, backoff_factor: float = None
) -> None:
if retries is None:
limit = 0
elif isinstance(retries, int):
limit = retries
else:
assert isinstance(retries, Retries)
Expand Down Expand Up @@ -423,6 +427,5 @@ def get_delays(self) -> typing.Iterator[float]:


DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0)
DEFAULT_RETRIES_CONFIG = Retries(0, backoff_factor=0.2)
DEFAULT_POOL_LIMITS = PoolLimits(soft_limit=10, hard_limit=100)
DEFAULT_MAX_REDIRECTS = 20
13 changes: 12 additions & 1 deletion tests/client/test_retries.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ async def test_no_retries() -> None:


@pytest.mark.usefixtures("async_environment")
async def test_retries() -> None:
async def test_client_retries() -> None:
client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=3)

response = await client.get("https://example.com")
Expand All @@ -102,6 +102,17 @@ async def test_retries() -> None:
assert response.status_code == 200


@pytest.mark.usefixtures("async_environment")
async def test_request_retries() -> None:
client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3))
response = await client.get("https://example.com", retries=3)
assert response.status_code == 200

client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=3), retries=3)
with pytest.raises(httpx.ConnectTimeout):
await client.get("https://example.com/connect_timeout", retries=None)


@pytest.mark.usefixtures("async_environment")
async def test_too_many_retries() -> None:
client = httpx.AsyncClient(dispatch=MockDispatch(succeed_after=2), retries=1)
Expand Down

0 comments on commit d3609c3

Please sign in to comment.