From bf6596a6617745fcae09610b3670561e0029bb38 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:53:40 +0100 Subject: [PATCH] refactor(datasets): Move `NetCDFDataset` to experimental (#739) * Move NetCDFDataset to experimental Signed-off-by: Ankita Katiyar * release notes and fix reqs Signed-off-by: Ankita Katiyar * Docs test Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar --- kedro-datasets/RELEASE.md | 2 ++ kedro-datasets/docs/source/api/kedro_datasets.rst | 1 - .../docs/source/api/kedro_datasets_experimental.rst | 1 + .../netcdf/__init__.py | 0 .../netcdf/netcdf_dataset.py | 0 .../tests}/__init__.py | 0 .../tests/netcdf/__init__.py | 0 .../tests/netcdf/test_netcdf_dataset.py | 2 +- kedro-datasets/pyproject.toml | 12 ++++++------ 9 files changed, 10 insertions(+), 8 deletions(-) rename kedro-datasets/{kedro_datasets => kedro_datasets_experimental}/netcdf/__init__.py (100%) rename kedro-datasets/{kedro_datasets => kedro_datasets_experimental}/netcdf/netcdf_dataset.py (100%) rename kedro-datasets/{tests/netcdf => kedro_datasets_experimental/tests}/__init__.py (100%) create mode 100644 kedro-datasets/kedro_datasets_experimental/tests/netcdf/__init__.py rename kedro-datasets/{ => kedro_datasets_experimental}/tests/netcdf/test_netcdf_dataset.py (99%) diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md index 88064ba5d..666b83a58 100755 --- a/kedro-datasets/RELEASE.md +++ b/kedro-datasets/RELEASE.md @@ -9,6 +9,8 @@ | `langchain.ChatCohereDataset` | A dataset for loading a ChatCohere langchain model. | `kedro_datasets_experimental.langchain` | | `langchain.OpenAIEmbeddingsDataset` | A dataset for loading a OpenAIEmbeddings langchain model. | `kedro_datasets_experimental.langchain` | | `langchain.ChatOpenAIDataset` | A dataset for loading a ChatOpenAI langchain model. | `kedro_datasets_experimental.langchain` | +| `netcdf.NetCDFDataset` | A dataset for loading and saving "*.nc" files. | `kedro_datasets_experimental.netcdf` | +* `netcdf.NetCDFDataset` moved from `kedro_datasets` to `kedro_datasets_experimental`. * Extended preview feature to `yaml.YAMLDataset`. ## Community contributions diff --git a/kedro-datasets/docs/source/api/kedro_datasets.rst b/kedro-datasets/docs/source/api/kedro_datasets.rst index be7c8d886..6d4047b53 100644 --- a/kedro-datasets/docs/source/api/kedro_datasets.rst +++ b/kedro-datasets/docs/source/api/kedro_datasets.rst @@ -24,7 +24,6 @@ kedro_datasets kedro_datasets.json.JSONDataset kedro_datasets.matlab.MatlabDataset kedro_datasets.matplotlib.MatplotlibWriter - kedro_datasets.netcdf.NetCDFDataset kedro_datasets.networkx.GMLDataset kedro_datasets.networkx.GraphMLDataset kedro_datasets.networkx.JSONDataset diff --git a/kedro-datasets/docs/source/api/kedro_datasets_experimental.rst b/kedro-datasets/docs/source/api/kedro_datasets_experimental.rst index ca276cedd..fbae09589 100644 --- a/kedro-datasets/docs/source/api/kedro_datasets_experimental.rst +++ b/kedro-datasets/docs/source/api/kedro_datasets_experimental.rst @@ -15,3 +15,4 @@ kedro_datasets_experimental kedro_datasets_experimental.langchain.ChatCohereDataset kedro_datasets_experimental.langchain.ChatOpenAIDataset kedro_datasets_experimental.langchain.OpenAIEmbeddingsDataset + kedro_datasets_experimental.netcdf.NetCDFDataset diff --git a/kedro-datasets/kedro_datasets/netcdf/__init__.py b/kedro-datasets/kedro_datasets_experimental/netcdf/__init__.py similarity index 100% rename from kedro-datasets/kedro_datasets/netcdf/__init__.py rename to kedro-datasets/kedro_datasets_experimental/netcdf/__init__.py diff --git a/kedro-datasets/kedro_datasets/netcdf/netcdf_dataset.py b/kedro-datasets/kedro_datasets_experimental/netcdf/netcdf_dataset.py similarity index 100% rename from kedro-datasets/kedro_datasets/netcdf/netcdf_dataset.py rename to kedro-datasets/kedro_datasets_experimental/netcdf/netcdf_dataset.py diff --git a/kedro-datasets/tests/netcdf/__init__.py b/kedro-datasets/kedro_datasets_experimental/tests/__init__.py similarity index 100% rename from kedro-datasets/tests/netcdf/__init__.py rename to kedro-datasets/kedro_datasets_experimental/tests/__init__.py diff --git a/kedro-datasets/kedro_datasets_experimental/tests/netcdf/__init__.py b/kedro-datasets/kedro_datasets_experimental/tests/netcdf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kedro-datasets/tests/netcdf/test_netcdf_dataset.py b/kedro-datasets/kedro_datasets_experimental/tests/netcdf/test_netcdf_dataset.py similarity index 99% rename from kedro-datasets/tests/netcdf/test_netcdf_dataset.py rename to kedro-datasets/kedro_datasets_experimental/tests/netcdf/test_netcdf_dataset.py index c2120b126..1526f89a5 100644 --- a/kedro-datasets/tests/netcdf/test_netcdf_dataset.py +++ b/kedro-datasets/kedro_datasets_experimental/tests/netcdf/test_netcdf_dataset.py @@ -8,7 +8,7 @@ from s3fs import S3FileSystem from xarray.testing import assert_equal -from kedro_datasets.netcdf import NetCDFDataset +from kedro_datasets_experimental.netcdf import NetCDFDataset FILE_NAME = "test.nc" MULTIFILE_NAME = "test*.nc" diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml index 3e4200612..d9eede037 100644 --- a/kedro-datasets/pyproject.toml +++ b/kedro-datasets/pyproject.toml @@ -80,9 +80,6 @@ matlab = ["kedro-datasets[matlab-matlabdataset]"] matplotlib-matplotlibwriter = ["matplotlib>=3.0.3, <4.0"] matplotlib = ["kedro-datasets[matplotlib-matplotlibwriter]"] -netcdf-netcdfdataset = ["h5netcdf>=1.2.0","netcdf4>=1.6.4","xarray>=2023.1.0"] -netcdf = ["kedro-datasets[netcdf-netcdfdataset]"] - networkx-gmldataset = ["kedro-datasets[networkx-base]"] networkx-graphmldataset = ["kedro-datasets[networkx-base]"] networkx-jsondataset = ["kedro-datasets[networkx-base]"] @@ -170,6 +167,8 @@ langchain-openaiembeddingsdataset = ["langchain-openai~=0.1.7"] langchain-chatanthropicdataset = ["langchain-anthropic~=0.1.13", "langchain-community~=0.2.0"] langchain-chatcoheredataset = ["langchain-cohere~=0.1.5", "langchain-community~=0.2.0"] langchain = ["kedro-datasets[langchain-chatopenaidataset,langchain-openaiembeddingsdataset,langchain-chatanthropicdataset,langchain-chatcoheredataset ]"] +netcdf-netcdfdataset = ["h5netcdf>=1.2.0","netcdf4>=1.6.4","xarray>=2023.1.0"] +netcdf = ["kedro-datasets[netcdf-netcdfdataset]"] # Docs requirements docs = [ @@ -198,7 +197,6 @@ test = [ "geopandas>=0.6.0, <1.0", "hdfs>=2.5.8, <3.0", "holoviews>=1.13.0", - "h5netcdf>=1.2.0", "ibis-framework[duckdb,examples]", "import-linter[toml]==1.2.6", "ipython>=7.31.1, <8.0", @@ -212,7 +210,6 @@ test = [ "memory_profiler>=0.50.0, <1.0", "moto==5.0.0", "mypy~=1.0", - "netcdf4>=1.6.4", "networkx~=2.4", "opencv-python~=4.5.5.64", "openpyxl>=3.0.3, <4.0", @@ -270,6 +267,9 @@ experimental = [ "langchain-cohere", "langchain-anthropic", "langchain-community", + "h5netcdf>=1.2.0", + "netcdf4>=1.6.4", + "xarray>=2023.1.0", ] # All requirements @@ -291,7 +291,7 @@ version = {attr = "kedro_datasets.__version__"} fail_under = 100 show_missing = true # temporarily ignore kedro_datasets/__init__.py in coverage report -omit = ["tests/*", "kedro_datasets/holoviews/*", "kedro_datasets/netcdf/*", "kedro_datasets/snowflake/*", "kedro_datasets/tensorflow/*", "kedro_datasets/__init__.py", "kedro_datasets/conftest.py"] +omit = ["tests/*", "kedro_datasets/holoviews/*", "kedro_datasets/snowflake/*", "kedro_datasets/tensorflow/*", "kedro_datasets/__init__.py", "kedro_datasets/conftest.py"] exclude_also = ["raise NotImplementedError", "if TYPE_CHECKING:"] [tool.pytest.ini_options]