Support for conditional requests (#188)

* Basic version of conditional requests.

* Some fixes.

* Improve github example.

* Some cleanups, add documentation.

* Add future annotations to avoid type issues with older python versions.

* Add test for conditional requests.

* Make flake happy.

* Use typing.Tuple.

* AsyncMock is not available for python 3.7.

* Addd proper decorator to skip tests for py37.

* Add test case when conditional requests are not supported.

aiohttp_client_cache/backends/base.py

@@ -108,6 +108,7 @@ async def request(
         method: str,
         url: StrOrURL,
         expire_after: ExpirationTime = None,
+        refresh: bool = False,
         **kwargs,
     ) -> Tuple[Optional[CachedResponse], CacheActions]:
         """Fetch a cached response based on request info
@@ -117,13 +118,16 @@ async def request(
             url: Request URL
             expire_after: Expiration time to set only for this request; overrides
                 ``CachedSession.expire_after``, and accepts all the same values.
+            refresh: Revalidate with the server before using a cached response, and refresh if needed
+                (e.g., a "soft refresh", like F5 in a browser)
             kwargs: All other request arguments
         """
         key = self.create_key(method, url, **kwargs)
         actions = CacheActions.from_request(
             key,
             url=url,
             request_expire_after=expire_after,
+            refresh=refresh,
             session_expire_after=self.expire_after,
             urls_expire_after=self.urls_expire_after,
             cache_control=self.cache_control,

aiohttp_client_cache/cache_control.py

@@ -1,4 +1,6 @@
 """Utilities for determining cache expiration and other cache actions"""
+from __future__ import annotations
+
 from datetime import datetime, timedelta, timezone
 from email.utils import parsedate_to_datetime
 from fnmatch import fnmatch
@@ -59,18 +61,21 @@ def from_request(
         key: str,
         cache_control: bool = False,
         cache_disabled: bool = False,
+        refresh: bool = False,
         headers: Optional[Mapping] = None,
         **kwargs,
     ):
         """Initialize from request info and CacheBackend settings"""
         if cache_disabled:
-            return cls(key=key, skip_read=True, skip_write=True, revalidate=True)
+            return cls(key=key, skip_read=True, skip_write=True)
         else:
             headers = headers or {}
             if cache_control and has_cache_headers(headers):
                 return cls.from_headers(key, headers)
             else:
-                return cls.from_settings(key, cache_control=cache_control, **kwargs)
+                return cls.from_settings(
+                    key, cache_control=cache_control, refresh=refresh, **kwargs
+                )
 
     @classmethod
     def from_headers(cls, key: str, headers: Mapping):
@@ -83,7 +88,7 @@ def from_headers(cls, key: str, headers: Mapping):
             expire_after=directives.get('max-age'),
             skip_read=do_not_cache or 'no-store' in directives or 'no-cache' in directives,
             skip_write=do_not_cache or 'no-store' in directives,
-            revalidate=do_not_cache,
+            revalidate=False,
         )
 
     @classmethod
@@ -92,6 +97,7 @@ def from_settings(
         key: str,
         url: StrOrURL,
         cache_control: bool = False,
+        refresh: bool = False,
         request_expire_after: ExpirationTime = None,
         session_expire_after: ExpirationTime = None,
         urls_expire_after: Optional[ExpirationPatterns] = None,
@@ -112,7 +118,7 @@ def from_settings(
             expire_after=expire_after,
             skip_read=do_not_cache,
             skip_write=do_not_cache,
-            revalidate=do_not_cache,
+            revalidate=refresh and not do_not_cache,
         )
 
     @property
@@ -190,6 +196,25 @@ def has_cache_headers(headers: Mapping) -> bool:
     return any([d in cache_control for d in CACHE_DIRECTIVES] + [bool(headers.get('Expires'))])
 
 
+def get_refresh_headers(
+    request_headers: Optional[Mapping], cached_headers: Mapping
+) -> tuple[bool, Mapping]:
+    """Returns headers containing directives for conditional requests if the cached headers support it.**"""
+    headers = request_headers if request_headers is not None else {}
+    refresh_headers = {k: v for k, v in headers.items()}
+    conditional_request_supported = False
+
+    if "ETag" in cached_headers:
+        refresh_headers["If-None-Match"] = cached_headers["ETag"]
+        conditional_request_supported = True
+
+    if "Last-Modified" in cached_headers:
+        refresh_headers["If-Modified-Since"] = cached_headers["Last-Modified"]
+        conditional_request_supported = True
+
+    return conditional_request_supported, refresh_headers
+
+
 def parse_http_date(value: str) -> Optional[datetime]:
     """Attempt to parse an HTTP (RFC 5322-compatible) timestamp"""
     try:

aiohttp_client_cache/session.py

@@ -2,14 +2,14 @@
 import warnings
 from contextlib import asynccontextmanager
 from logging import getLogger
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Tuple
 
 from aiohttp import ClientSession
 from aiohttp.typedefs import StrOrURL
 
 from aiohttp_client_cache.backends import CacheBackend, get_valid_kwargs
-from aiohttp_client_cache.cache_control import ExpirationTime
-from aiohttp_client_cache.response import AnyResponse, set_response_defaults
+from aiohttp_client_cache.cache_control import CacheActions, ExpirationTime, get_refresh_headers
+from aiohttp_client_cache.response import AnyResponse, CachedResponse, set_response_defaults
 from aiohttp_client_cache.signatures import extend_signature
 
 if TYPE_CHECKING:
@@ -39,19 +39,37 @@ def __init__(
 
     @extend_signature(ClientSession._request)
     async def _request(
-        self, method: str, str_or_url: StrOrURL, expire_after: ExpirationTime = None, **kwargs
+        self,
+        method: str,
+        str_or_url: StrOrURL,
+        expire_after: ExpirationTime = None,
+        refresh: bool = False,
+        **kwargs,
     ) -> AnyResponse:
         """Wrapper around :py:meth:`.SessionClient._request` that adds caching"""
         # Attempt to fetch cached response
         response, actions = await self.cache.request(
-            method, str_or_url, expire_after=expire_after, **kwargs
+            method, str_or_url, expire_after=expire_after, refresh=refresh, **kwargs
         )
 
+        def restore_cookies(r):
+            self.cookie_jar.update_cookies(r.cookies or {}, r.url)
+            for redirect in r.history:
+                self.cookie_jar.update_cookies(redirect.cookies or {}, redirect.url)
+
+        if actions.revalidate and response:
+            from_cache, new_response = await self._refresh_cached_response(
+                method, str_or_url, response, actions, **kwargs
+            )
+            if not from_cache:
+                return set_response_defaults(new_response)
+            else:
+                restore_cookies(new_response)
+                return new_response
+
         # Restore any cached cookies to the session
         if response:
-            self.cookie_jar.update_cookies(response.cookies or {}, response.url)
-            for redirect in response.history:
-                self.cookie_jar.update_cookies(redirect.cookies or {}, redirect.url)
+            restore_cookies(response)
             return response
         # If the response was missing or expired, send and cache a new request
         else:
@@ -65,6 +83,48 @@ async def _request(
                 await self.cache.save_response(new_response, actions.key, actions.expires)
             return set_response_defaults(new_response)
 
+    async def _refresh_cached_response(
+        self,
+        method: str,
+        str_or_url: StrOrURL,
+        cached_response: CachedResponse,
+        actions: CacheActions,
+        **kwargs,
+    ) -> Tuple[bool, AnyResponse]:
+        """Checks if the cached response is still valid using conditional requests if supported"""
+
+        # check whether we can do a conditional request,
+        # i.e. if the necessary headers are present (ETag, Last-Modified)
+        conditional_request_supported, refresh_headers = get_refresh_headers(
+            kwargs.get("headers", None), cached_response.headers
+        )
+
+        if conditional_request_supported:
+            logger.debug(f'Refreshing cached response; making request to {str_or_url}')
+            refreshed_response = await super()._request(
+                method, str_or_url, headers=refresh_headers, **kwargs
+            )
+
+            if refreshed_response.status == 304:
+                logger.debug('Cached response not modified; returning cached response')
+                return True, cached_response
+            else:
+                actions.update_from_response(refreshed_response)
+                if await self.cache.is_cacheable(refreshed_response, actions):
+                    logger.debug('Cached response refreshed; updating cache')
+                    await self.cache.save_response(refreshed_response, actions.key, actions.expires)
+                else:
+                    logger.debug('Cached response refreshed; deleting from cache')
+                    await self.cache.delete(actions.key)
+
+                return False, refreshed_response
+        else:
+            logger.debug(
+                'Conditional requests not supported, no ETag or Last-Modified headers present; '
+                'returning cached response'
+            )
+            return True, cached_response
+
     async def close(self):
         """Close both aiohttp connector and any backend connection(s) on contextmanager exit"""
         await super().close()

examples/github.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# fmt: off
+"""
+An example of making conditional requests to the GitHub Rest API`
+"""
+
+import asyncio
+import logging
+
+from aiohttp_client_cache import CachedSession, FileBackend
+
+CACHE_DIR = "~/.cache/aiohttp-requests"
+
+
+async def main():
+    cache = FileBackend(cache_name=CACHE_DIR, use_temp=True)
+    await cache.clear()
+
+    org = "requests-cache"
+    url = f"https://api.github.com/orgs/{org}/repos"
+
+    # we make 2 requests for the same resource (list of all repos of the requests-cache organization)
+    # the second request refreshes the cached response with the remote server
+    # the debug output should illustrate that the cached response gets refreshed
+    async with CachedSession(cache=cache) as session:
+        response = await session.get(url)
+        print(f"url = {response.url}, status = {response.status}, "
+              f"ratelimit-used = {response.headers['x-ratelimit-used']}")
+
+        await asyncio.sleep(1)
+
+        response = await session.get(url, refresh=True)
+        print(f"url = {response.url}, status = {response.status}, "
+              f"ratelimit-used = {response.headers['x-ratelimit-used']}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger("aiohttp_client_cache").setLevel(logging.DEBUG)
+    asyncio.run(main())

test/integration/base_backend_test.py

@@ -341,3 +341,51 @@ async def test_disabled(self):
 
                 assert response.from_cache is False
                 assert await session.cache.responses.size() == 1
+
+    @skip_37
+    async def test_conditional_request(self):
+        """Test that conditional requests using refresh=True work.
+        The `/cache` endpoint returns proper ETag header and responds to a request
+        with an If-None-Match header with a 304 response.
+        """
+
+        async with self.init_session() as session:
+            # mock the _refresh_cached_response method to verify
+            # that a conditional request is being made
+            from unittest.mock import AsyncMock
+
+            mock_refresh = AsyncMock(wraps=session._refresh_cached_response)
+            session._refresh_cached_response = mock_refresh
+
+            response = await session.get(httpbin('cache'))
+            assert response.from_cache is False
+            mock_refresh.assert_not_awaited()
+
+            response = await session.get(httpbin('cache'), refresh=True)
+            assert response.from_cache is True
+            mock_refresh.assert_awaited_once()
+
+    @skip_37
+    async def test_no_support_for_conditional_request(self):
+        """Test that conditional requests using refresh=True work even when the
+        cached response / server does not support conditional requests. In this case
+        the cached response shall be returned as if no refresh=True option would
+        have been passed in.
+        The `/cache/<int>` endpoint returns no ETag header and just returns a normal 200 response.
+        """
+
+        async with self.init_session() as session:
+            # mock the _refresh_cached_response method to verify
+            # that a conditional request is being made
+            from unittest.mock import AsyncMock
+
+            mock_refresh = AsyncMock(wraps=session._refresh_cached_response)
+            session._refresh_cached_response = mock_refresh
+
+            response = await session.get(httpbin('cache/10'))
+            assert response.from_cache is False
+            mock_refresh.assert_not_awaited()
+
+            response = await session.get(httpbin('cache/10'), refresh=True)
+            assert response.from_cache is True
+            mock_refresh.assert_awaited_once()