From 127b930f948082f35a33ee107e3ec94b9c9efbd6 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 28 Apr 2020 09:35:34 +0200 Subject: [PATCH 1/9] Lazy regridding with Linear, Nearest, AreaWeighted --- lib/iris/_lazy_data.py | 36 +++++++++++++++++++++++++++++++++ lib/iris/analysis/_regrid.py | 27 ++++++++++++++++--------- lib/iris/experimental/regrid.py | 13 ++++++++++-- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/lib/iris/_lazy_data.py b/lib/iris/_lazy_data.py index be4c1d4a0d..6611305ece 100644 --- a/lib/iris/_lazy_data.py +++ b/lib/iris/_lazy_data.py @@ -349,3 +349,39 @@ def lazy_elementwise(lazy_array, elementwise_op): dtype = elementwise_op(np.zeros(1, lazy_array.dtype)).dtype return da.map_blocks(elementwise_op, lazy_array, dtype=dtype) + + +def _map_complete_blocks(src, func, dims, out_sizes): + """Apply a function to complete blocks. + + Complete means that the data is not chunked along the chosen dimensions. + + Args: + + * src (:class:`~iris.cube.Cube`): + Source cube that function is applied to. + * func: + Function to apply. + * dims (tuple of int): + Dimensions that cannot be chunked. + * out_sizes (tuple of int): + Output size of dimensions that cannot be chunked. + + """ + if not src.has_lazy_data(): + return func(src.data) + + data = src.lazy_data() + + # Ensure dims are not chunked + in_chunks = list(data.chunks) + for dim in dims: + in_chunks[dim] = src.shape[dim] + data = data.rechunk(in_chunks) + + # Determine output chunks + out_chunks = list(data.chunks) + for dim, size in zip(dims, out_sizes): + out_chunks[dim] = size + + return data.map_blocks(func, chunks=out_chunks, dtype=src.dtype) diff --git a/lib/iris/analysis/_regrid.py b/lib/iris/analysis/_regrid.py index 71584f04c0..eccfdb1139 100644 --- a/lib/iris/analysis/_regrid.py +++ b/lib/iris/analysis/_regrid.py @@ -11,6 +11,7 @@ import numpy as np import numpy.ma as ma +from iris._lazy_data import _map_complete_blocks from iris.analysis._interpolation import ( EXTRAPOLATION_MODES, extend_circular_coord_and_data, @@ -1040,16 +1041,22 @@ def __call__(self, src): # Compute the interpolated data values. x_dim = src.coord_dims(src_x_coord)[0] y_dim = src.coord_dims(src_y_coord)[0] - data = self._regrid( - src.data, - x_dim, - y_dim, - src_x_coord, - src_y_coord, - sample_grid_x, - sample_grid_y, - self._method, - self._extrapolation_mode, + + # Define regrid function + regrid = functools.partial( + self._regrid, + x_dim=x_dim, + y_dim=y_dim, + src_x_coord=src_x_coord, + src_y_coord=src_y_coord, + sample_grid_x=sample_grid_x, + sample_grid_y=sample_grid_y, + method=self._method, + extrapolation_mode=self._extrapolation_mode, + ) + + data = _map_complete_blocks( + src, regrid, (y_dim, x_dim), sample_grid_x.shape ) # Wrap up the data as a Cube. diff --git a/lib/iris/experimental/regrid.py b/lib/iris/experimental/regrid.py index e79e03cf72..1460c70e17 100644 --- a/lib/iris/experimental/regrid.py +++ b/lib/iris/experimental/regrid.py @@ -18,6 +18,7 @@ import numpy.ma as ma import scipy.interpolate +from iris._lazy_data import _map_complete_blocks import iris.analysis.cartography from iris.analysis._interpolation import ( get_xy_dim_coords, @@ -931,8 +932,16 @@ def _regrid_area_weighted_rectilinear_src_and_grid__perform( ) = regrid_info # Calculate new data array for regridded cube. - new_data = _regrid_area_weighted_array( - src_cube.data, src_x_dim, src_y_dim, weights_info, mdtol, + regrid = functools.partial( + _regrid_area_weighted_array, + x_dim=src_x_dim, + y_dim=src_y_dim, + weights_info=weights_info, + mdtol=mdtol, + ) + + new_data = _map_complete_blocks( + src_cube, regrid, (src_y_dim, src_x_dim), meshgrid_x.shape ) # Wrap up the data as a Cube. From 7ba9739ee0b7f53f50ed4882b2c951104a1b9e28 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 26 Aug 2020 17:20:29 +0200 Subject: [PATCH 2/9] Add tests --- lib/iris/_lazy_data.py | 2 +- lib/iris/analysis/_regrid.py | 4 +- lib/iris/experimental/regrid.py | 4 +- ..._area_weighted_rectilinear_src_and_grid.py | 10 +- .../regrid/test_RectilinearRegridder.py | 32 +++++++ .../lazy_data/test_map_complete_blocks.py | 93 +++++++++++++++++++ 6 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 lib/iris/tests/unit/lazy_data/test_map_complete_blocks.py diff --git a/lib/iris/_lazy_data.py b/lib/iris/_lazy_data.py index 6611305ece..dcc0674fe6 100644 --- a/lib/iris/_lazy_data.py +++ b/lib/iris/_lazy_data.py @@ -351,7 +351,7 @@ def lazy_elementwise(lazy_array, elementwise_op): return da.map_blocks(elementwise_op, lazy_array, dtype=dtype) -def _map_complete_blocks(src, func, dims, out_sizes): +def map_complete_blocks(src, func, dims, out_sizes): """Apply a function to complete blocks. Complete means that the data is not chunked along the chosen dimensions. diff --git a/lib/iris/analysis/_regrid.py b/lib/iris/analysis/_regrid.py index eccfdb1139..673f0f7e0e 100644 --- a/lib/iris/analysis/_regrid.py +++ b/lib/iris/analysis/_regrid.py @@ -11,7 +11,7 @@ import numpy as np import numpy.ma as ma -from iris._lazy_data import _map_complete_blocks +from iris._lazy_data import map_complete_blocks from iris.analysis._interpolation import ( EXTRAPOLATION_MODES, extend_circular_coord_and_data, @@ -1055,7 +1055,7 @@ def __call__(self, src): extrapolation_mode=self._extrapolation_mode, ) - data = _map_complete_blocks( + data = map_complete_blocks( src, regrid, (y_dim, x_dim), sample_grid_x.shape ) diff --git a/lib/iris/experimental/regrid.py b/lib/iris/experimental/regrid.py index 1460c70e17..d173246a6f 100644 --- a/lib/iris/experimental/regrid.py +++ b/lib/iris/experimental/regrid.py @@ -18,7 +18,7 @@ import numpy.ma as ma import scipy.interpolate -from iris._lazy_data import _map_complete_blocks +from iris._lazy_data import map_complete_blocks import iris.analysis.cartography from iris.analysis._interpolation import ( get_xy_dim_coords, @@ -940,7 +940,7 @@ def _regrid_area_weighted_rectilinear_src_and_grid__perform( mdtol=mdtol, ) - new_data = _map_complete_blocks( + new_data = map_complete_blocks( src_cube, regrid, (src_y_dim, src_x_dim), meshgrid_x.shape ) diff --git a/lib/iris/tests/experimental/regrid/test_regrid_area_weighted_rectilinear_src_and_grid.py b/lib/iris/tests/experimental/regrid/test_regrid_area_weighted_rectilinear_src_and_grid.py index 7f8ad4b551..56f7bebc4b 100644 --- a/lib/iris/tests/experimental/regrid/test_regrid_area_weighted_rectilinear_src_and_grid.py +++ b/lib/iris/tests/experimental/regrid/test_regrid_area_weighted_rectilinear_src_and_grid.py @@ -15,6 +15,7 @@ import copy import random +import dask.array as da import numpy as np import numpy.ma as ma @@ -456,7 +457,14 @@ def test_ten_by_ten_subset(self): indices = tuple([slice(i, i + 10), slice(j, j + 10)]) dest = src[indices] res = regrid_area_weighted(src, dest) - self.assertTrue(res, src[indices]) + self.assertEqual(res, src[indices]) + + def test_lazy_nop(self): + src = self.realistic_cube[:2, :3, :10, :10] + src.data = da.asarray(src.data, chunks=((1, 1), (2, 1), (10,), (10,))) + res = regrid_area_weighted(src, src) + self.assertTrue(res.has_lazy_data()) + self.assertEqual(res, src) def test_cross_section(self): # Slice to get a cross section. diff --git a/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py b/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py index 492283f843..f1e385731a 100644 --- a/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py +++ b/lib/iris/tests/unit/analysis/regrid/test_RectilinearRegridder.py @@ -9,6 +9,7 @@ # importing anything else. import iris.tests as tests +import dask.array as da import numpy as np import numpy.ma as ma @@ -471,6 +472,24 @@ def test_method_result_types(self): self.assertEqual(result_dtypes, expected_types_mapping) +class Test___call___lazy(tests.IrisTest): + def setUp(self): + self.cube = lat_lon_cube() + # Regridder method and extrapolation-mode. + self.args = ("linear", "mask") + self.regridder = Regridder(self.cube, self.cube, *self.args) + self.lazy_cube = self.cube.copy(da.asarray(self.cube.data)) + self.lazy_regridder = Regridder( + self.lazy_cube, self.lazy_cube, *self.args + ) + + def test_lazy_regrid(self): + result = self.lazy_regridder(self.lazy_cube) + self.assertTrue(result.has_lazy_data()) + expected = self.regridder(self.cube) + self.assertTrue(result == expected) + + class Test___call____invalid_types(tests.IrisTest): def setUp(self): self.cube = lat_lon_cube() @@ -1230,6 +1249,9 @@ def setUp(self): # The destination grid points are exactly the same as the # src grid points. self.src = realistic_4d()[:5, :2, ::40, ::30] + self.lazy_src = self.src.copy( + da.asarray(self.src.data, chunks=(1, 2) + self.src.shape[2:]) + ) self.grid = self.src.copy() def test_nop__linear(self): @@ -1242,6 +1264,16 @@ def test_nop__nearest(self): result = regridder(self.src) self.assertEqual(result, self.src) + def test_nop__linear_lazy(self): + regridder = Regridder(self.lazy_src, self.grid, "linear", "mask") + result = regridder(self.lazy_src) + self.assertEqual(result, self.lazy_src) + + def test_nop__nearest_lazy(self): + regridder = Regridder(self.lazy_src, self.grid, "nearest", "mask") + result = regridder(self.lazy_src) + self.assertEqual(result, self.lazy_src) + @tests.skip_data class Test___call____circular(tests.IrisTest): diff --git a/lib/iris/tests/unit/lazy_data/test_map_complete_blocks.py b/lib/iris/tests/unit/lazy_data/test_map_complete_blocks.py new file mode 100644 index 0000000000..15462ce53f --- /dev/null +++ b/lib/iris/tests/unit/lazy_data/test_map_complete_blocks.py @@ -0,0 +1,93 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +"""Test function :func:`iris._lazy data.map_complete_blocks`.""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import unittest +import numpy as np +import dask.array as da + +from iris._lazy_data import is_lazy_data, map_complete_blocks + + +def create_mock_cube(array): + cube = unittest.mock.Mock() + cube_data = unittest.mock.PropertyMock(return_value=array) + type(cube).data = cube_data + cube.dtype = array.dtype + cube.has_lazy_data = unittest.mock.Mock(return_value=is_lazy_data(array)) + cube.lazy_data = unittest.mock.Mock(return_value=array) + cube.shape = array.shape + return cube, cube_data + + +class Test_map_complete_blocks(tests.IrisTest): + def setUp(self): + self.array = np.arange(8).reshape(2, 4) + self.func = lambda chunk: chunk + 1 + self.func_result = self.array + 1 + + def test_non_lazy_input(self): + # Check that a non-lazy input doesn't trip up the functionality. + cube, cube_data = create_mock_cube(self.array) + result = map_complete_blocks( + cube, self.func, dims=(1,), out_sizes=(4,) + ) + self.assertFalse(is_lazy_data(result)) + self.assertArrayEqual(result, self.func_result) + # check correct data was accessed + cube.lazy_data.assert_not_called() + cube_data.assert_called_once() + + def test_lazy_input(self): + lazy_array = da.asarray(self.array, chunks=((1, 1), (4,))) + cube, cube_data = create_mock_cube(lazy_array) + result = map_complete_blocks( + cube, self.func, dims=(1,), out_sizes=(4,) + ) + self.assertTrue(is_lazy_data(result)) + self.assertArrayEqual(result.compute(), self.func_result) + # check correct data was accessed + cube.lazy_data.assert_called_once() + cube_data.assert_not_called() + + def test_rechunk(self): + lazy_array = da.asarray(self.array, chunks=((1, 1), (2, 2))) + cube, _ = create_mock_cube(lazy_array) + result = map_complete_blocks( + cube, self.func, dims=(1,), out_sizes=(4,) + ) + self.assertTrue(is_lazy_data(result)) + self.assertArrayEqual(result.compute(), self.func_result) + + def test_different_out_shape(self): + lazy_array = da.asarray(self.array, chunks=((1, 1), (4,))) + cube, _ = create_mock_cube(lazy_array) + + def func(_): + return np.arange(2).reshape(1, 2) + + func_result = [[0, 1], [0, 1]] + result = map_complete_blocks(cube, func, dims=(1,), out_sizes=(2,)) + self.assertTrue(is_lazy_data(result)) + self.assertArrayEqual(result.compute(), func_result) + + def test_multidimensional_input(self): + array = np.arange(2 * 3 * 4).reshape(2, 3, 4) + lazy_array = da.asarray(array, chunks=((1, 1), (1, 2), (4,))) + cube, _ = create_mock_cube(lazy_array) + result = map_complete_blocks( + cube, self.func, dims=(1, 2), out_sizes=(3, 4) + ) + self.assertTrue(is_lazy_data(result)) + self.assertArrayEqual(result.compute(), array + 1) + + +if __name__ == "__main__": + tests.main() From 867b29690e8ad847f300d4516eba417448c8bb60 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 26 Aug 2020 17:37:42 +0200 Subject: [PATCH 3/9] Add documentation --- lib/iris/analysis/_area_weighted.py | 8 ++++++++ lib/iris/analysis/_regrid.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/iris/analysis/_area_weighted.py b/lib/iris/analysis/_area_weighted.py index 7ff5430ca6..14cad3e5aa 100644 --- a/lib/iris/analysis/_area_weighted.py +++ b/lib/iris/analysis/_area_weighted.py @@ -78,6 +78,9 @@ def __call__(self, cube): The given cube must be defined with the same grid as the source grid used to create this :class:`AreaWeightedRegridder`. + If the source cube has lazy data, the returned cube will also + have lazy data. + Args: * cube: @@ -89,6 +92,11 @@ def __call__(self, cube): this cube will be converted to values on the new grid using area-weighted regridding. + .. note:: + + If the source cube has lazy data, chunks in the horizontal + dimensions will be combined before regridding. + """ src_x, src_y = get_xy_dim_coords(cube) if (src_x, src_y) != self._src_grid: diff --git a/lib/iris/analysis/_regrid.py b/lib/iris/analysis/_regrid.py index 673f0f7e0e..55762468ba 100644 --- a/lib/iris/analysis/_regrid.py +++ b/lib/iris/analysis/_regrid.py @@ -985,6 +985,9 @@ def __call__(self, src): The given cube must be defined with the same grid as the source grid used to create this :class:`RectilinearRegridder`. + If the source cube has lazy data, the returned cube will also + have lazy data. + Args: * src: @@ -996,6 +999,11 @@ def __call__(self, src): this cube will be converted to values on the new grid using either nearest-neighbour or linear interpolation. + .. note:: + + If the source cube has lazy data, chunks in the horizontal + dimensions will be combined before regridding. + """ # Validity checks. if not isinstance(src, iris.cube.Cube): From c1851a0c79c28e48ac8ce7de743d64368d1424cd Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Sep 2020 09:44:32 +0200 Subject: [PATCH 4/9] Add whatsnew message --- .../newfeature_2020-Sep-03_lazy_regridding.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt new file mode 100644 index 0000000000..802e42596c --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt @@ -0,0 +1 @@ +* Lazy regridding with the Linear, Nearest, and AreaWeighted regridding schemes. From 26b2adbbb15111c3cd44c75b7b0fb3ac7f0c2a0b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Sep 2020 13:49:22 +0200 Subject: [PATCH 5/9] Link to classes in whatsnew entry Co-authored-by: Patrick Peglar --- .../newfeature_2020-Sep-03_lazy_regridding.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt index 802e42596c..b710eedaea 100644 --- a/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt +++ b/docs/iris/src/whatsnew/contributions_3.0.0/newfeature_2020-Sep-03_lazy_regridding.txt @@ -1 +1 @@ -* Lazy regridding with the Linear, Nearest, and AreaWeighted regridding schemes. +* Lazy regridding with the :class:`~iris.analysis.Linear`, :class:`~iris.analysis.Nearest`, and :class:`~iris.analysis.AreaWeighted` regridding schemes. From 56ba9b72ec926d7e767ce28a47604fa51962cd77 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Sep 2020 16:41:45 +0200 Subject: [PATCH 6/9] Clarify laziness status and add example --- .../interpolation_and_regridding.rst | 46 +++++++++++++++++++ .../iris/src/userguide/real_and_lazy_data.rst | 4 +- lib/iris/analysis/__init__.py | 16 +++++++ lib/iris/analysis/_area_weighted.py | 5 +- lib/iris/analysis/_regrid.py | 8 +++- lib/iris/cube.py | 17 +++++-- lib/iris/experimental/regrid.py | 4 ++ 7 files changed, 91 insertions(+), 9 deletions(-) diff --git a/docs/iris/src/userguide/interpolation_and_regridding.rst b/docs/iris/src/userguide/interpolation_and_regridding.rst index 65ac36eada..72645e611f 100644 --- a/docs/iris/src/userguide/interpolation_and_regridding.rst +++ b/docs/iris/src/userguide/interpolation_and_regridding.rst @@ -28,6 +28,10 @@ The following are the regridding schemes that are currently available in Iris: * nearest-neighbour regridding (:class:`iris.analysis.Nearest`), and * area-weighted regridding (:class:`iris.analysis.AreaWeighted`, first-order conservative). +These regridding schemes support lazy regridding, i.e. if the source cube has +lazy data, the resulting cube will also have lazy data. +See :doc:`real_and_lazy_data` for an introduction to lazy data. + .. _interpolation: @@ -409,3 +413,45 @@ regridded to the target grid. For example:: In each case ``result`` will be the input cube regridded to the grid defined by the target grid cube (in this case ``rotated_psl``) that we used to define the cached regridder. + +Regridding lazy data +^^^^^^^^^^^^^^^^^^^^ + +If you are working with large cubes, especially when you are regridding to a +high resolution target grid, you may run out of memory when trying to +regrid a cube. When this happens, make sure the input cube has lazy data + + >>> air_temp = iris.load_cube(iris.sample_data_path('A1B_north_america.nc')) + >>> air_temp + [] + >>> air_temp.has_lazy_data() + True + +and the regridding scheme supports lazy data. All regridding schemes described +here support lazy data. If you still run out of memory even while using lazy +data, inspect the +`chunks `__ +: + + >>> air_temp.lazy_data().chunks + ((240,), (37,), (49,)) + +The cube above consist of a single chunk, because it is fairly small. For +larger cubes, iris will automatically create chunks of an optimal size when +loading the data. However, because regridding to a high resolution grid +may dramatically increase the size of the data, the automatically chosen +chunks might be too large. + +As an example of how to solve this, we could manually re-chunk the time +dimension, to regrid it in 8 chunks of 30 timesteps at a time: + + >>> air_temp.data = air_temp.lazy_data().rechunk([30, None, None]) + >>> air_temp.lazy_data().chunks + ((30, 30, 30, 30, 30, 30, 30, 30), (37,), (49,)) + +Assuming that Dask is configured such that it processes only a few chunks of +the data array at a time, this will further reduce memory use. + +Note that chunking in the horizontal dimensions is not supported by the +regridding schemes. Chunks in these dimensions will automatically be combined +before regridding. diff --git a/docs/iris/src/userguide/real_and_lazy_data.rst b/docs/iris/src/userguide/real_and_lazy_data.rst index 84a35efa64..a58114de73 100644 --- a/docs/iris/src/userguide/real_and_lazy_data.rst +++ b/docs/iris/src/userguide/real_and_lazy_data.rst @@ -37,7 +37,9 @@ In Iris, lazy data is provided as a `dask array `_. A dask array also has a shape and data type but the dask array's data points remain on disk and only loaded into memory in -small chunks when absolutely necessary. This has key performance benefits for +small +`chunks `__ +when absolutely necessary. This has key performance benefits for handling large amounts of data, where both calculation time and storage requirements can be significantly reduced. diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index a1e56533fd..a049d060c2 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -2440,6 +2440,10 @@ def regridder(self, src_grid, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Supports lazy regridding. Any + `chunks `__ + in horizontal dimensions will be combined before regridding. + Args: * src_grid: @@ -2514,6 +2518,10 @@ def regridder(self, src_grid_cube, target_grid_cube): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Supports lazy regridding. Any + `chunks `__ + in horizontal dimensions will be combined before regridding. + Args: * src_grid_cube: @@ -2630,6 +2638,10 @@ def regridder(self, src_grid, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Supports lazy regridding. Any + `chunks `__ + in horizontal dimensions will be combined before regridding. + Args: * src_grid: @@ -2716,6 +2728,8 @@ def regridder(self, src_cube, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Does not support lazy regridding. + Args: * src_cube: @@ -2791,6 +2805,8 @@ def regridder(self, src_grid, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Does not support lazy regridding. + Args: * src_grid: diff --git a/lib/iris/analysis/_area_weighted.py b/lib/iris/analysis/_area_weighted.py index 14cad3e5aa..6872800215 100644 --- a/lib/iris/analysis/_area_weighted.py +++ b/lib/iris/analysis/_area_weighted.py @@ -94,8 +94,9 @@ def __call__(self, cube): .. note:: - If the source cube has lazy data, chunks in the horizontal - dimensions will be combined before regridding. + If the source cube has lazy data, + `chunks `__ + in the horizontal dimensions will be combined before regridding. """ src_x, src_y = get_xy_dim_coords(cube) diff --git a/lib/iris/analysis/_regrid.py b/lib/iris/analysis/_regrid.py index 55762468ba..1036c3028d 100644 --- a/lib/iris/analysis/_regrid.py +++ b/lib/iris/analysis/_regrid.py @@ -439,6 +439,9 @@ def __call__(self, src): The given cube must be defined with the same grid as the source grid used to create this :class:`_CurvilinearRegridder`. + If the source cube has lazy data, it will be realized before + regridding and the returned cube will also have realized data. + Args: * src: @@ -1001,8 +1004,9 @@ def __call__(self, src): .. note:: - If the source cube has lazy data, chunks in the horizontal - dimensions will be combined before regridding. + If the source cube has lazy data, + `chunks `__ + in the horizontal dimensions will be combined before regridding. """ # Validity checks. diff --git a/lib/iris/cube.py b/lib/iris/cube.py index cc833f8848..0ede0ee6df 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4448,7 +4448,7 @@ def interpolate(self, sample_points, scheme, collapse_scalar=True): return interp(points, collapse_scalar=collapse_scalar) def regrid(self, grid, scheme): - """ + r""" Regrid this :class:`~iris.cube.Cube` on to the given target `grid` using the given regridding `scheme`. @@ -4461,9 +4461,15 @@ def regrid(self, grid, scheme): target grid. The regridding schemes currently available in Iris are: - * :class:`iris.analysis.Linear`, - * :class:`iris.analysis.Nearest`, and - * :class:`iris.analysis.AreaWeighted`. + * :class:`iris.analysis.Linear`\*, + * :class:`iris.analysis.Nearest`\*, + * :class:`iris.analysis.AreaWeighted`\*, + * :class:`iris.analysis.UnstructuredNearest`, + * :class:`iris.analysis.PointInCell`, + * :class:`iris.experimental.regrid.ProjectedUnstructuredLinear`, + * :class:`iris.experimental.regrid.ProjectedUnstructuredNearest`. + + \* Supports lazy regridding. Returns: A cube defined with the horizontal dimensions of the target grid @@ -4471,6 +4477,9 @@ def regrid(self, grid, scheme): this cube will be converted to values on the new grid according to the given regridding scheme. + The returned cube will have lazy data if the original cube has + lazy data and the regridding scheme supports lazy regridding. + .. note:: Both the source and target cubes must have a CoordSystem, otherwise diff --git a/lib/iris/experimental/regrid.py b/lib/iris/experimental/regrid.py index d173246a6f..e08b71c403 100644 --- a/lib/iris/experimental/regrid.py +++ b/lib/iris/experimental/regrid.py @@ -1479,6 +1479,8 @@ def regridder(self, src_cube, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Does not support lazy regridding. + Args: * src_cube: @@ -1544,6 +1546,8 @@ def regridder(self, src_cube, target_grid): constructing your own regridder is preferable. These are detailed in the :ref:`user guide `. + Does not support lazy regridding. + Args: * src_cube: From b2a50a423ecaf733245e7d8aca35c1ce6a6e317d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Sep 2020 17:10:41 +0200 Subject: [PATCH 7/9] Fix example output --- docs/iris/src/userguide/interpolation_and_regridding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/iris/src/userguide/interpolation_and_regridding.rst b/docs/iris/src/userguide/interpolation_and_regridding.rst index 72645e611f..d8f3a3d484 100644 --- a/docs/iris/src/userguide/interpolation_and_regridding.rst +++ b/docs/iris/src/userguide/interpolation_and_regridding.rst @@ -423,7 +423,7 @@ regrid a cube. When this happens, make sure the input cube has lazy data >>> air_temp = iris.load_cube(iris.sample_data_path('A1B_north_america.nc')) >>> air_temp - [] + >>> air_temp.has_lazy_data() True From ef8206ae5c7260967b9e0d3d5a7ca06bf1215284 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 7 Sep 2020 20:28:23 +0200 Subject: [PATCH 8/9] Remove mention of experimental regridding schemes. --- lib/iris/cube.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 0ede0ee6df..c2d4b2ab36 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4466,8 +4466,6 @@ def regrid(self, grid, scheme): * :class:`iris.analysis.AreaWeighted`\*, * :class:`iris.analysis.UnstructuredNearest`, * :class:`iris.analysis.PointInCell`, - * :class:`iris.experimental.regrid.ProjectedUnstructuredLinear`, - * :class:`iris.experimental.regrid.ProjectedUnstructuredNearest`. \* Supports lazy regridding. From f524d9acd9dcc25425f313277093bdb47d74b270 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 7 Sep 2020 20:41:37 +0200 Subject: [PATCH 9/9] Clarify documentation --- docs/iris/src/userguide/interpolation_and_regridding.rst | 5 +++-- lib/iris/cube.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/iris/src/userguide/interpolation_and_regridding.rst b/docs/iris/src/userguide/interpolation_and_regridding.rst index d8f3a3d484..ffed21a7f5 100644 --- a/docs/iris/src/userguide/interpolation_and_regridding.rst +++ b/docs/iris/src/userguide/interpolation_and_regridding.rst @@ -28,8 +28,9 @@ The following are the regridding schemes that are currently available in Iris: * nearest-neighbour regridding (:class:`iris.analysis.Nearest`), and * area-weighted regridding (:class:`iris.analysis.AreaWeighted`, first-order conservative). -These regridding schemes support lazy regridding, i.e. if the source cube has -lazy data, the resulting cube will also have lazy data. +The linear, nearest-neighbor, and area-weighted regridding schemes support +lazy regridding, i.e. if the source cube has lazy data, the resulting cube +will also have lazy data. See :doc:`real_and_lazy_data` for an introduction to lazy data. diff --git a/lib/iris/cube.py b/lib/iris/cube.py index c2d4b2ab36..0a27c2f0e7 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4458,8 +4458,7 @@ def regrid(self, grid, scheme): A :class:`~iris.cube.Cube` that defines the target grid. * scheme: The type of regridding to use to regrid this cube onto the - target grid. The regridding schemes currently available - in Iris are: + target grid. The regridding schemes in Iris currently include: * :class:`iris.analysis.Linear`\*, * :class:`iris.analysis.Nearest`\*,