Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a test for usability of duck arrays with chunks property #8739

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions xarray/namedarray/parallelcompat.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,13 @@ def guess_chunkmanager(
)


def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint[Any]:
def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint[Any] | None:
"""
Detects which parallel backend should be used for given set of arrays.

Also checks that all arrays are of same chunking type (i.e. not a mix of cubed and dask).

Returns None if no matching ChunkManager is found.
"""

# TODO this list is probably redundant with something inside xarray.apply_ufunc
Expand Down Expand Up @@ -155,9 +157,7 @@ def get_chunked_array_type(*args: Any) -> ChunkManagerEntrypoint[Any]:
if chunkmanager.is_chunked_array(chunked_arr)
]
if not selected:
raise TypeError(
f"Could not find a Chunk Manager which recognises type {type(chunked_arr)}"
)
return None
elif len(selected) >= 2:
raise TypeError(f"Multiple ChunkManagers recognise type {type(chunked_arr)}")
else:
Expand Down
8 changes: 5 additions & 3 deletions xarray/namedarray/pycompat.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ def to_numpy(
# TODO first attempt to call .to_numpy() once some libraries implement it
if is_chunked_array(data):
chunkmanager = get_chunked_array_type(data)
data, *_ = chunkmanager.compute(data, **kwargs)
if chunkmanager is not None:
data, *_ = chunkmanager.compute(data, **kwargs)
if isinstance(data, array_type("cupy")):
data = data.get()
# pint has to be imported dynamically as pint imports xarray
Expand All @@ -127,8 +128,9 @@ def to_duck_array(data: Any, **kwargs: dict[str, Any]) -> duckarray[_ShapeType,

if is_chunked_array(data):
chunkmanager = get_chunked_array_type(data)
loaded_data, *_ = chunkmanager.compute(data, **kwargs) # type: ignore[var-annotated]
return loaded_data
if chunkmanager is not None:
loaded_data, *_ = chunkmanager.compute(data, **kwargs) # type: ignore[var-annotated]
return loaded_data

if isinstance(data, ExplicitlyIndexed):
return data.get_duck_array() # type: ignore[no-untyped-call, no-any-return]
Expand Down
5 changes: 1 addition & 4 deletions xarray/tests/test_parallelcompat.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,7 @@ def test_raise_if_no_arrays_chunked(self, register_dummy_chunkmanager) -> None:
def test_raise_if_no_matching_chunkmanagers(self) -> None:
dummy_arr = DummyChunkedArray([1, 2, 3])

with pytest.raises(
TypeError, match="Could not find a Chunk Manager which recognises"
):
get_chunked_array_type(dummy_arr)
assert get_chunked_array_type(dummy_arr) is None

@requires_dask
def test_detect_dask_if_installed(self) -> None:
Expand Down
23 changes: 22 additions & 1 deletion xarray/tests/test_variable.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from xarray.core.types import T_DuckArray
from xarray.core.utils import NDArrayMixin
from xarray.core.variable import as_compatible_data, as_variable
from xarray.namedarray.pycompat import array_type
from xarray.namedarray.pycompat import array_type, is_chunked_array
from xarray.tests import (
assert_allclose,
assert_array_equal,
Expand Down Expand Up @@ -2722,6 +2722,27 @@ def __init__(self, array):
orig = Variable(dims=(), data=array)
assert isinstance(orig._data.item(), CustomWithValuesAttr)

def test_duck_array_with_chunks(self):
# Non indexable type
class CustomArray(NDArrayMixin):
def __init__(self, array):
self.array = array

@property
def chunks(self):
return self.shape

def __array_function__(self, *args, **kwargs):
return NotImplemented

def __array_ufunc__(self, *args, **kwargs):
return NotImplemented

array = CustomArray(np.arange(3))
assert is_chunked_array(array)
var = Variable(dims=("x"), data=array)
var.load()


def test_raise_no_warning_for_nan_in_binary_ops():
with assert_no_warnings():
Expand Down
Loading