Add optional file-based listings caching

docs/source/api.rst

@@ -47,7 +47,9 @@ Base Classes
    fsspec.core.OpenFiles
    fsspec.core.get_fs_token_paths
    fsspec.core.url_to_fs
-   fsspec.dircache.DirCache
+   fsspec.dircache.DisabledListingsCache
+   fsspec.dircache.MemoryListingsCache
+   fsspec.dircache.FileListingsCache
    fsspec.FSMap
    fsspec.generic.GenericFileSystem
    fsspec.registry.register_implementation
@@ -82,7 +84,13 @@ Base Classes
 
 .. autofunction:: fsspec.core.url_to_fs
 
-.. autoclass:: fsspec.dircache.DirCache
+.. autoclass:: fsspec.dircache.DisabledListingsCache
+   :members: __init__
+
+.. autoclass:: fsspec.dircache.MemoryListingsCache
+   :members: __init__
+
+.. autoclass:: fsspec.dircache.FileListingsCache
    :members: __init__
 
 .. autoclass:: fsspec.FSMap

docs/source/changelog.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+Dev
+--------
+
+Enhancements
+
+- add file-based listing cache using diskcache (#895)
+  warning: use new ``listings_cache_options`` instead of ``use_listings_cache`` etc.
+
 2024.3.1
 --------
 

docs/source/features.rst

@@ -181,15 +181,26 @@ Listings Caching
 ----------------
 
 For some implementations, getting file listings (i.e., ``ls`` and anything that
-depends on it) is expensive. These implementations use dict-like instances of
-:class:`fsspec.dircache.DirCache` to manage the listings.
-
-The cache allows for time-based expiry of entries with the ``listings_expiry_time``
-parameter, or LRU expiry with the ``max_paths`` parameter. These can be
-set on any implementation instance that uses listings caching; or to skip the
-caching altogether, use ``use_listings_cache=False``. That would be appropriate
-when the target location is known to be volatile because it is being written
-to from other sources.
+depends on it) is expensive. These implementations maye use either dict-like instances of
+:class:`fsspec.dircache.MemoryListingsCache` or file-based caching with instances of
+:class:`fsspec.dircache.FileListingsCache` to manage the listings.
+
+The listings cache can be controlled via the keyword ``listings_cache_options`` which is a dictionary.
+The type of cache that is used, can be controlled via the keyword ``cache_type`` (`disabled`, `memory` or `file`).
+The cache allows for time-based expiry of entries with the keyword ``expiry_time``. If the target location is known to
+be volatile because e.g. it is being written to from other sources we recommend to disable the listings cache.
+If you want to use the file-based caching, you can also provide the argument
+``directory`` to determine where the cache file is stored.
+
+Example for ``listings_cache_options``:
+
+.. code-block:: json
+
+        {
+            "cache_type": "file",
+            "expiry_time": 3600,
+            "directory": "/tmp/cache"
+        }
 
 When the ``fsspec`` instance writes to the backend, the method ``invalidate_cache``
 is called, so that subsequent listing of the given paths will force a refresh. In

fsspec/archive.py

@@ -37,19 +37,19 @@ def _all_dirnames(self, paths):
     def info(self, path, **kwargs):
         self._get_dirs()
         path = self._strip_protocol(path)
-        if path in {"", "/"} and self.dir_cache:
+        if path in {"", "/"} and self.dircache:
             return {"name": "", "type": "directory", "size": 0}
-        if path in self.dir_cache:
-            return self.dir_cache[path]
-        elif path + "/" in self.dir_cache:
-            return self.dir_cache[path + "/"]
+        if path in self.dircache:
+            return self.dircache[path]
+        elif path + "/" in self.dircache:
+            return self.dircache[path + "/"]
         else:
             raise FileNotFoundError(path)
 
     def ls(self, path, detail=True, **kwargs):
         self._get_dirs()
         paths = {}
-        for p, f in self.dir_cache.items():
+        for p, f in self.dircache.items():
             p = p.rstrip("/")
             if "/" in p:
                 root = p.rsplit("/", 1)[0]

fsspec/asyn.py

@@ -312,15 +312,23 @@ class AsyncFileSystem(AbstractFileSystem):
     mirror_sync_methods = True
     disable_throttling = False
 
-    def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs):
+    def __init__(
+        self,
+        *args,
+        asynchronous=False,
+        loop=None,
+        batch_size=None,
+        listings_cache_options=None,
+        **kwargs,
+    ):
         self.asynchronous = asynchronous
         self._pid = os.getpid()
         if not asynchronous:
             self._loop = loop or get_loop()
         else:
             self._loop = None
         self.batch_size = batch_size
-        super().__init__(*args, **kwargs)
+        super().__init__(listings_cache_options, *args, **kwargs)
 
     @property
     def loop(self):

fsspec/implementations/http.py

@@ -3,7 +3,6 @@
 import logging
 import re
 import weakref
-from copy import copy
 from urllib.parse import urlparse
 
 import aiohttp
@@ -58,6 +57,7 @@ def __init__(
         client_kwargs=None,
         get_client=get_client,
         encoded=False,
+        listings_cache_options=None,
         **storage_options,
     ):
         """
@@ -83,11 +83,39 @@ def __init__(
             A callable which takes keyword arguments and constructs
             an aiohttp.ClientSession. It's state will be managed by
             the HTTPFileSystem class.
+        listings_cache_options: dict
+            Options for the listings cache.
         storage_options: key-value
             Any other parameters passed on to requests
         cache_type, cache_options: defaults used in open
         """
-        super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options)
+        # TODO: remove in future release
+        # Clean caching-related parameters from `storage_options`
+        # before propagating them as `request_options` through `self.kwargs`.
+        old_listings_cache_kwargs = {
+            "use_listings_cache",
+            "listings_expiry_time",
+            "max_paths",
+            "skip_instance_cache",
+        }
+        # intersection of old_listings_cache_kwargs and storage_options
+        old_listings_cache_kwargs = old_listings_cache_kwargs.intersection(
+            storage_options
+        )
+        if old_listings_cache_kwargs:
+            logger.warning(
+                f"The following parameters are not used anymore and will be ignored: {old_listings_cache_kwargs}. "
+                f"Use new `listings_cache_options` instead."
+            )
+            for key in old_listings_cache_kwargs:
+                del storage_options[key]
+        super().__init__(
+            self,
+            asynchronous=asynchronous,
+            loop=loop,
+            listings_cache_options=listings_cache_options,
+            **storage_options,
+        )
         self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE
         self.simple_links = simple_links
         self.same_schema = same_scheme
@@ -96,19 +124,10 @@ def __init__(
         self.client_kwargs = client_kwargs or {}
         self.get_client = get_client
         self.encoded = encoded
-        self.kwargs = storage_options
-        self._session = None
-
-        # Clean caching-related parameters from `storage_options`
-        # before propagating them as `request_options` through `self.kwargs`.
         # TODO: Maybe rename `self.kwargs` to `self.request_options` to make
         #       it clearer.
-        request_options = copy(storage_options)
-        self.use_listings_cache = request_options.pop("use_listings_cache", False)
-        request_options.pop("listings_expiry_time", None)
-        request_options.pop("max_paths", None)
-        request_options.pop("skip_instance_cache", None)
-        self.kwargs = request_options
+        self.kwargs = storage_options
+        self._session = None
 
     @property
     def fsid(self):
@@ -201,7 +220,7 @@ async def _ls_real(self, url, detail=True, **kwargs):
             return sorted(out)
 
     async def _ls(self, url, detail=True, **kwargs):
-        if self.use_listings_cache and url in self.dircache:
+        if url in self.dircache:
             out = self.dircache[url]
         else:
             out = await self._ls_real(url, detail=detail, **kwargs)

fsspec/implementations/libarchive.py

@@ -115,7 +115,7 @@ def __init__(
             Kwargs passed when instantiating the target FS, if ``fo`` is
             a string.
         """
-        super().__init__(self, **kwargs)
+        super().__init__(False, self, **kwargs)
         if mode != "r":
             raise ValueError("Only read from archive files accepted")
         if isinstance(fo, str):

fsspec/implementations/tar.py

@@ -82,7 +82,7 @@ def __init__(
         self._fo_ref = fo
         self.fo = fo  # the whole instance is a context
         self.tar = tarfile.TarFile(fileobj=self.fo)
-        self.dir_cache = None
+        self.dircache = None
 
         self.index_store = index_store
         self.index = None
@@ -101,19 +101,19 @@ def _index(self):
         # TODO: save index to self.index_store here, if set
 
     def _get_dirs(self):
-        if self.dir_cache is not None:
+        if self.dircache is not None:
             return
 
         # This enables ls to get directories as children as well as files
-        self.dir_cache = {
+        self.dircache = {
             dirname: {"name": dirname, "size": 0, "type": "directory"}
             for dirname in self._all_dirnames(self.tar.getnames())
         }
         for member in self.tar.getmembers():
             info = member.get_info()
             info["name"] = info["name"].rstrip("/")
             info["type"] = typemap.get(info["type"], "file")
-            self.dir_cache[info["name"]] = info
+            self.dircache[info["name"]] = info
 
     def _open(self, path, mode="rb", **kwargs):
         if mode != "rb":