Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle fsspec.FSMap using FSStore store #1304

Merged
merged 2 commits into from
Dec 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/release.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ Unreleased
* Fix bug that caused double counting of groups in ``groups()`` and ``group_keys()``
methods with V3 stores.
By :user:`Ryan Abernathey <rabernat>` :issue:`1228`.
* Handle fsspec.FSMap using FSStore store
By :user:`Rafal Wojdyla <ravwojdyla>` :issue:`1304`.

.. _release_2.13.2:

Expand Down
10 changes: 10 additions & 0 deletions zarr/_storage/v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,16 @@ def _normalize_store_arg_v3(store: Any, storage_options=None, mode="r") -> BaseS
return store
if isinstance(store, os.PathLike):
store = os.fspath(store)
if FSStore._fsspec_installed():
import fsspec
if isinstance(store, fsspec.FSMap):
return FSStoreV3(store.root,
fs=store.fs,
mode=mode,
check=store.check,
create=store.create,
missing_exceptions=store.missing_exceptions,
**(storage_options or {}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's actually fine to pass storage_options through like this. If it is not empty, it will trigger an error via

zarr-python/zarr/storage.py

Lines 1329 to 1331 in 1af77b6

else:
if storage_options:
raise ValueError("Cannot specify both fs and storage_options")

This is the desired behavior. Storage options will already be configured on the FSMap, so specifying them twice would not make sense.

if isinstance(store, str):
if "://" in store or "::" in store:
store = FSStoreV3(store, mode=mode, **(storage_options or {}))
Expand Down
19 changes: 19 additions & 0 deletions zarr/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,16 @@ def _normalize_store_arg_v2(store: Any, storage_options=None, mode="r") -> BaseS
return store
if isinstance(store, os.PathLike):
store = os.fspath(store)
if FSStore._fsspec_installed():
import fsspec
if isinstance(store, fsspec.FSMap):
return FSStore(store.root,
fs=store.fs,
mode=mode,
check=store.check,
create=store.create,
missing_exceptions=store.missing_exceptions,
**(storage_options or {}))
if isinstance(store, str):
if "://" in store or "::" in store:
return FSStore(store, mode=mode, **(storage_options or {}))
Expand Down Expand Up @@ -1308,6 +1318,8 @@ def __init__(self, url, normalize_keys=False, key_separator=None,
create=False,
missing_exceptions=None,
**storage_options):
if not self._fsspec_installed(): # pragma: no cover
raise ImportError("`fsspec` is required to use zarr's FSStore")
import fsspec

mapper_options = {"check": check, "create": create}
Expand Down Expand Up @@ -1479,6 +1491,13 @@ def clear(self):
raise ReadOnlyError()
self.map.clear()

@classmethod
def _fsspec_installed(cls):
"""Returns true if fsspec is installed"""
import importlib.util

return importlib.util.find_spec("fsspec") is not None
ravwojdyla marked this conversation as resolved.
Show resolved Hide resolved


class TempStore(DirectoryStore):
"""Directory store using a temporary directory for storage.
Expand Down
5 changes: 5 additions & 0 deletions zarr/tests/test_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -2556,10 +2556,15 @@ def test_normalize_store_arg(tmpdir):
assert isinstance(store, Class)

if have_fsspec:
import fsspec

path = tempfile.mkdtemp()
store = normalize_store_arg("file://" + path, zarr_version=2, mode='w')
assert isinstance(store, FSStore)

store = normalize_store_arg(fsspec.get_mapper("file://" + path))
assert isinstance(store, FSStore)


def test_meta_prefix_6853():

Expand Down
5 changes: 5 additions & 0 deletions zarr/tests/test_storage_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -467,11 +467,16 @@ def test_normalize_store_arg_v3(tmpdir):
normalize_store_arg(str(fn), zarr_version=3, mode='w', storage_options={"some": "kwargs"})

if have_fsspec:
import fsspec

path = tempfile.mkdtemp()
store = normalize_store_arg("file://" + path, zarr_version=3, mode='w')
assert isinstance(store, FSStoreV3)
assert 'zarr.json' in store

store = normalize_store_arg(fsspec.get_mapper("file://" + path), zarr_version=3)
assert isinstance(store, FSStoreV3)

fn = tmpdir.join('store.n5')
with pytest.raises(NotImplementedError):
normalize_store_arg(str(fn), zarr_version=3, mode='w')
Expand Down