From 2dfcd0fb6bc253ca83c3ba93a8422494a1c8b77c Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:09:24 -0400 Subject: [PATCH 01/67] update pre-commit hooks --- .pre-commit-config.yaml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b4c6f0f..b5360c61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,31 +6,33 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: [ --py38-plus ] + args: [ '--py38-plus' ] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: trailing-whitespace - exclude: .rvc|.rvh|.rvi|.rvp|.rvt|.tpl|.txt|setup.cfg - id: end-of-file-fixer - exclude: .ipynb|.rvc|.rvh|.rvi|.rvp|.rvt|.tpl|.txt - id: check-json + - id: check-toml - id: check-yaml - args: [ --allow-multiple-documents ] + args: [ '--allow-multiple-documents' ] + - id: debug-statements - id: pretty-format-json args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ] - - id: debug-statements - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - id: python-check-blanket-noqa - id: rst-inline-touching-normal + - repo: https://github.com/pappasam/toml-sort + rev: v0.23.0 + hooks: + - id: toml-sort-fix - repo: https://github.com/psf/black rev: 23.3.0 hooks: - id: black exclude: ^docs/ - args: [ --target-version=py38 ] - repo: https://github.com/pycqa/flake8 rev: 6.0.0 hooks: @@ -40,19 +42,16 @@ repos: rev: 5.12.0 hooks: - id: isort - args: [ --settings-file=setup.cfg ] - repo: https://github.com/nbQA-dev/nbQA rev: 1.7.0 hooks: - id: nbqa-pyupgrade args: [ --py38-plus ] - additional_dependencies: [ pyupgrade==3.3.1 ] + additional_dependencies: [ 'pyupgrade==3.3.1' ] - id: nbqa-black - args: [ --target-version=py38 ] - additional_dependencies: [ black==23.3.0 ] + additional_dependencies: [ 'black==23.3.0' ] - id: nbqa-isort - args: [ --settings-file=setup.cfg ] - additional_dependencies: [ isort==5.12.0 ] + additional_dependencies: [ 'isort==5.12.0' ] - repo: https://github.com/pycqa/pydocstyle rev: 6.3.0 hooks: @@ -62,16 +61,12 @@ repos: rev: v0.3.8 hooks: - id: blackdoc - additional_dependencies: [ black==23.3.0 ] + additional_dependencies: [ 'black==23.3.0' ] - repo: https://github.com/adrienverge/yamllint.git rev: v1.30.0 hooks: - id: yamllint - args: [ '--config-file', '.yamllint.yaml' ] - - repo: https://github.com/mgedmin/check-manifest - rev: "0.49" - hooks: - - id: check-manifest + args: [ '--config-file=.yamllint.yaml' ] - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.22.0 hooks: From c1f547dca1c4a75562b843a9d018d6b179f02e1f Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:09:37 -0400 Subject: [PATCH 02/67] remove MANIFEST.in --- MANIFEST.in | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a55c7dcb..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,36 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst -include requirements_dev.txt -include requirements_docs.txt -include requirements_gis.txt - -recursive-include ravenpy *.py *.zip *.csv *.rst -recursive-include tests * -recursive-include benchmark *.ipynb *.txt -recursive-include docs *.rst *.md conf.py Makefile make.bat *.jpg *.png *.gif *.ipynb - -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] -recursive-exclude docs/notebooks/.ipynb_checkpoints * -recursive-exclude docs/_build * -recursive-exclude docs/apidoc *.rst -recursive-exclude docs/.jupyter_cache * -recursive-exclude docs/jupyter_execute * - -graft ravenpy/data - -exclude .coveralls.yml -exclude .cruft.json -exclude .editorconfig -exclude .pre-commit-config.yaml -exclude .readthedocs.yml -exclude .yamllint.yaml -exclude .zenodo.json -exclude Makefile -exclude environment.yml -exclude environment-rtd.yml -exclude mypy.ini -exclude tox.ini From d41b6e4d32e57647694d2acca056cdd96e628668 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:10:04 -0400 Subject: [PATCH 03/67] remove requirement files --- requirements_dev.txt | 21 --------------------- requirements_docs.txt | 31 ------------------------------- requirements_gis.txt | 9 --------- 3 files changed, 61 deletions(-) delete mode 100644 requirements_dev.txt delete mode 100644 requirements_docs.txt delete mode 100644 requirements_gis.txt diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index b0fdacf5..00000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,21 +0,0 @@ -pip>=21.0,<23.1 -bump2version -wheel -watchdog -flake8 -tox -coverage -coveralls -Sphinx -twine -Click -pytest -pytest-cov -pytest-xdist>=3.2.0 -filelock -black>=23.3.0 -isort -pre-commit -holoviews -hvplot -setuptools<65.6 diff --git a/requirements_docs.txt b/requirements_docs.txt deleted file mode 100644 index dcdca91e..00000000 --- a/requirements_docs.txt +++ /dev/null @@ -1,31 +0,0 @@ -sphinx -sphinx-click -sphinx-codeautolink -sphinx-copybutton -sphinx-rtd-theme>=1.0 -nbsphinx -pandoc -ipython -ipykernel -jupyter_client -sphinx-click -numpydoc -pymetalink -autodoc_pydantic -owslib<0.29 -birdhouse-birdy -intake -intake-esm -ipyleaflet -ipywidgets -jupyter -s3fs -clisops -intake-xarray -gcsfs -hs_restclient -jupytext -jupyter-cache -myst_nb -cartopy -xesmf diff --git a/requirements_gis.txt b/requirements_gis.txt deleted file mode 100644 index 92c5f079..00000000 --- a/requirements_gis.txt +++ /dev/null @@ -1,9 +0,0 @@ -affine -fiona>=1.9 -geopandas>=0.9.0 -lxml -owslib>=0.24.1 -pyproj>=3.0.0 -rasterio -rioxarray -shapely From 5b25eba9f432a85504d396b907dfb0ca065f1461 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:10:38 -0400 Subject: [PATCH 04/67] update dynamic package fields --- ravenpy/__init__.py | 2 +- ravenpy/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ravenpy/__init__.py b/ravenpy/__init__.py index 97cf1ca7..b7586a95 100644 --- a/ravenpy/__init__.py +++ b/ravenpy/__init__.py @@ -1,4 +1,4 @@ -"""Top-level package for RavenPy.""" +"""A Python package to help run Raven, the hydrologic modelling framework.""" from .__version__ import __author__, __email__, __version__ # noqa: F401 from .ravenpy import Emulator, EnsembleReader, OutputReader, RavenWarning, run diff --git a/ravenpy/__version__.py b/ravenpy/__version__.py index 5648c599..0e695d02 100644 --- a/ravenpy/__version__.py +++ b/ravenpy/__version__.py @@ -2,6 +2,6 @@ # without importing the main package when its dependencies are not installed. # See: https://packaging.python.org/guides/single-sourcing-package-version -__author__ = """David Huard""" +__author__ = "David Huard" __email__ = "huard.david@ouranos.ca" __version__ = "0.11.0" From 102870feda0d62f7fb9b654eb27579e97b6b1385 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:11:24 -0400 Subject: [PATCH 05/67] update setup.cfg for new structure --- setup.cfg | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/setup.cfg b/setup.cfg index 16d3f78f..8c170481 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,10 +3,6 @@ current_version = 0.11.0 commit = True tag = False -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - [bumpversion:file:ravenpy/__version__.py] search = __version__ = "{current_version}" replace = __version__ = "{new_version}" @@ -15,32 +11,6 @@ replace = __version__ = "{new_version}" search = "version": "{current_version}", replace = "version": "{new_version}", -[aliases] -test = pytest - -[tool:pytest] -collect_ignore = - setup.py -addopts = - --color=yes - --verbose - --numprocesses=0 - --maxprocesses=8 - --dist=worksteal -python_files = test_*.py -norecursedirs = src .git bin -filterwarnings = - ignore::UserWarning - -[isort] -profile = black -py_version = 38 -append_only = true - -[coverage:run] -relative_files = True -omit = tests/* - [flake8] exclude = .git, @@ -83,6 +53,16 @@ rst-roles = py:ref, ref +[pycodestyle] +count = False +exclude = tests +ignore = + E226, + E402, + E501, +max-line-length = 120 +statistics = True + [pydocstyle] convention = numpy -match = ((?!(test_|conf)).)*\.py +match = ((?!(test_|conftest|conf)).)*\.py From d93926093715696c23c1ea0630b9a18cdc25080c Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:26:05 -0400 Subject: [PATCH 06/67] use relative imports to deal with import resolution issues --- ravenpy/config/commands.py | 3 +-- ravenpy/config/parsers.py | 3 +-- ravenpy/config/rvs.py | 7 +++---- ravenpy/ravenpy.py | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ravenpy/config/commands.py b/ravenpy/config/commands.py index 4bdd23cf..6b156bd2 100644 --- a/ravenpy/config/commands.py +++ b/ravenpy/config/commands.py @@ -29,8 +29,7 @@ validator, ) -from ravenpy.config import options - +from ..config import options from .base import ( Command, FlatCommand, diff --git a/ravenpy/config/parsers.py b/ravenpy/config/parsers.py index cbfd0fb1..548f08eb 100644 --- a/ravenpy/config/parsers.py +++ b/ravenpy/config/parsers.py @@ -5,8 +5,7 @@ import cftime from xarray import open_dataset -from ravenpy.config import commands as rc - +from ..config import commands as rc from .conventions import RAVEN_OUTPUT_FMT diff --git a/ravenpy/config/rvs.py b/ravenpy/config/rvs.py index 7051099e..83fde0dc 100644 --- a/ravenpy/config/rvs.py +++ b/ravenpy/config/rvs.py @@ -6,10 +6,9 @@ import cftime from pydantic import Field, root_validator, validator -from ravenpy.config import commands as rc -from ravenpy.config import options as o -from ravenpy.config import processes as rp - +from ..config import commands as rc +from ..config import options as o +from ..config import processes as rp from .base import RV, Sym, parse_symbolic """ diff --git a/ravenpy/ravenpy.py b/ravenpy/ravenpy.py index f59a58ce..5459f817 100644 --- a/ravenpy/ravenpy.py +++ b/ravenpy/ravenpy.py @@ -12,8 +12,8 @@ import xarray as xr -from ravenpy.config import parsers -from ravenpy.config.rvs import Config +from .config import parsers +from .config.rvs import Config RAVEN_EXEC_PATH = os.getenv("RAVENPY_RAVEN_BINARY_PATH") or shutil.which("raven") From 2dece386ee73bbf5828f50b314079ba6548c54e8 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:27:53 -0400 Subject: [PATCH 07/67] replace setup.py with pyproject.toml --- pyproject.toml | 222 +++++++++++++++++++++++++++++++++++++ setup.py | 291 ------------------------------------------------- 2 files changed, 222 insertions(+), 291 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ad939e0a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,222 @@ +[build-system] +requires = ["flit_core >=3.8,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "ravenpy" +authors = [ + {name = "David Huard", email = "huard.david@ouranos.ca"} +] +maintainers = [ + {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"} +] +readme = {file = "README.rst", content-type = "text/x-rst"} +requires-python = ">=3.8.0" +keywords = ["raven", "raven-hydro", "hydrology", "gis", "analysis", "modelling"] +license = {file = "LICENSE"} +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Hydrology" +] +dynamic = ["description", "version"] +dependencies = [ + "cftime", + "cf-xarray", + "click", + "climpred>=2.1", + "dask", + "haversine", + "matplotlib", + "netCDF4", + "numpy", + "owslib<0.29", + "pandas<2.0", + "pint>=0.20", + "pydantic", + "pymbolic", + "requests", + "scipy", + "spotpy", + "statsmodels", + "wheel", + "xarray<2022.11.0", # Pinned due to incompatibility with climpred @ 2.2.0 + "xclim>=0.40.0", + "xskillscore" +] + +[project.optional-dependencies] +dev = [ + "Click", + "black>=23.3.0", + "bump2version", + "coverage", + "coveralls", + "filelock", + "flake8", + "holoviews", + "hvplot", + "isort", + "pre-commit", + "pytest", + "pytest-cov", + "pytest-xdist>=3.2.0", + "tox", + "watchdog", + "wheel", + # GIS Dependencies + "affine", + "fiona>=1.9", + "geopandas>=0.9.0", + "gdal", + "lxml", + "owslib>=0.24.1,<0.29", + "pyproj>=3.0.0", + "rasterio", + "rioxarray", + "shapely" +] +docs = [ + "autodoc_pydantic", + "birdhouse-birdy", + "cartopy", + "clisops", + "gcsfs", + "hs_restclient", + "intake", + "intake-esm", + "intake-xarray", + "ipykernel", + "ipyleaflet", + "ipython", + "ipywidgets", + "jupyter-cache", + "jupyter_client", + "jupytext", + "myst_nb", + "nbsphinx", + "numpydoc", + "owslib>=0.24.1,<0.29", + "pandoc", + "pymetalink", + "s3fs", + "sphinx", + "sphinx-click", + "sphinx-codeautolink", + "sphinx-copybutton", + "sphinx-rtd-theme>=1.0", + "xesmf", + # GIS Dependencies + "affine", + "fiona>=1.9", + "geopandas>=0.9.0", + "gdal", + "lxml", + "owslib>=0.24.1,<0.29", + "pyproj>=3.0.0", + "rasterio", + "rioxarray", + "shapely" +] +gis = [ + "affine", + "fiona>=1.9", + "geopandas>=0.9.0", + "gdal", + "lxml", + "owslib>=0.24.1,<0.29", + "pyproj>=3.0.0", + "rasterio", + "rioxarray", + "shapely" +] + +[project.scripts] +raven = "ravenpy.cli:main" + +[project.urls] +"Homepage" = "https://ravenpy.readthedocs.io" +"Source" = "https://github.com/CSHS-CWRA/RavenPy" +"Changelog" = "https://ravenpy.readthedocs.io/en/latest/history.html" +"Issue tracker" = "https://github.com/CSHS-CWRA/RavenPy/issues" +"About Ouranos" = "https://www.ouranos.ca/en/" +"About CSHS-CWRA" = "https://cwra.org/en/" + +[tool] + +[tool.black] +target-version = [ + "py38", + "py39", + "py310" +] + +[tool.coverage.run] +relative_files = true +omit = ["tests/*.py"] + +[tool.flit.sdist] +include = [ + "AUTHORS.rst", + "HISTORY.rst", + "CONTRIBUTING.rst", + "LICENSE", + "Makefile", + "README.rst", + "docs/**/*.ipynb", + "docs/**/*.md", + "docs/**/*.rst", + "docs/Makefile", + "docs/make.bat", + "docs/conf.py", + "environment*.yml", + "ravenpy/**/*.py", + "ravenpy/data/**/*.csv", + "ravenpy/data/**/*.zip", + "setup.cfg", + "tests/*.py", + "tests/test.cfg", + "tox.ini" +] +exclude = [ + "**/*.py[co]", + "**/__pycache__", + ".*", + "docs/_build", + "docs/apidoc/modules.rst", + "docs/source/**/.ipynb_checkpoints", + "docs/apidoc/ravenpy*.rst" +] + +[tool.isort] +profile = "black" +py_version = 38 +append_only = true + +[tool.pytest.ini_options] +addopts = [ + "--verbose", + "--color=yes", + "--strict-markers", + "--tb=native", + "--numprocesses=0", + "--maxprocesses=8", + "--dist=worksteal" +] +python_files = "test_*.py" +norecursedirs = ["src", ".git", "bin"] +filterwarnings = ["ignore::UserWarning"] +testpaths = "tests" diff --git a/setup.py b/setup.py deleted file mode 100644 index 202cfc1a..00000000 --- a/setup.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python - -"""The setup script.""" - -import os -import shutil -import subprocess -import urllib.request -import zipfile -from pathlib import Path -from typing import Optional, Union -from urllib.parse import urljoin - -# Note: setuptools < 65.6 is needed for some dependencies (see: https://github.com/pypa/setuptools/issues/3693) -from setuptools import Distribution, find_packages, setup -from setuptools.command.develop import develop -from setuptools.command.install import install - -RAVEN_VERSION = "3.6" - -with open("README.rst") as readme_file: - readme = readme_file.read() - -with open("HISTORY.rst") as history_file: - history = history_file.read() - -requirements = [ - "cftime", - "cf-xarray", - "click", - "climpred>=2.1", - "dask", - "haversine", - "matplotlib", - "netCDF4", - "numpy", - "owslib<0.29", - "pandas<2.0", - "pint>=0.20", - "pydantic", - "pymbolic", - "requests", - "scipy", - "spotpy", - "statsmodels", - "wheel", - "xarray<2022.11.0", # Pinned due to incompatibility with climpred @ 2.2.0 - "xclim>=0.40.0", - "xskillscore", -] - -test_requirements = [ - "pytest>=3", -] - -docs_requirements = [ - dependency for dependency in open("requirements_docs.txt").readlines() -] - -gis_requirements = [ - dependency for dependency in open("requirements_gis.txt").readlines() -] -# Special GDAL handling -on_conda = os.getenv("CONDA_BUILD") -if on_conda == "1": - gis_requirements.append("gdal") -else: - try: - gdal_version = subprocess.run( - ["gdal-config", "--version"], capture_output=True - ).stdout.decode("utf-8") - gis_requirements.append(f"gdal=={gdal_version}") - except (subprocess.CalledProcessError, FileNotFoundError): - pass - -dev_requirements = gis_requirements.copy() -dev_requirements.extend( - [dependency for dependency in open("requirements_dev.txt").readlines()] -) - - -# Idea taken from: https://stackoverflow.com/a/25176606/787842 -class OnlyGetScriptPath(install): - def run(self): - # does not call install.run() by design - self.distribution.install_scripts = self.install_scripts - - -def get_setuptools_install_scripts_dir(): - dist = Distribution({"cmdclass": {"install": OnlyGetScriptPath}}) - dist.dry_run = True # not sure if necessary, but to be safe - dist.parse_config_files() - command = dist.get_command_obj("install") - command.ensure_finalized() - command.run() - return dist.install_scripts - - -def create_external_deps_install_class(command_cls): - """ - Class factory command to implement the customized binary download + compile + install logic - for both the install and develop command contexts. - """ - - class InstallExternalDeps(command_cls): - """ - Custom handler for the 'install' and 'develop' commands, to download, extract and compile - the source code of Raven and copy the resulting binaries in a location - available on the PATH. - """ - - external_deps_path = None - with_binaries = False - - user_options = command_cls.user_options + [ - # The format is (long option, short option, description). - ( - "with-binaries", - None, - "Download Raven sources and compile them.", - ), - ] - - def initialize_options(self): - """Set default values for options.""" - # Each user option must be listed here with their default value. - command_cls.initialize_options(self) - self.with_binaries = False - - def finalize_options(self): - command_cls.finalize_options(self) - - def install_binary_dep( - self, - url, - name: str, - version: str, - rev_name: Optional[str] = None, - binary_name: str = "", - make_target: str = "", - src_folder: Optional[Union[str, os.PathLike]] = None, - remove_line: Optional[str] = None, - ): - print(f"Downloading {name} source code..") - if rev_name: - file_path = f"v{(Path(version) / rev_name).as_posix()}" - else: - file_path = f"v{version}" - - print( - f"{urljoin(url, file_path)}.zip", - self.external_deps_path / f"{name}.zip", - ) - urllib.request.urlretrieve( - f"{urljoin(url, file_path)}.zip", - self.external_deps_path / f"{name}.zip", - ) - - print(f"Extracting {name} source code..") - if rev_name: - out_folder = self.external_deps_path.joinpath(rev_name) - else: - out_folder = self.external_deps_path - with zipfile.ZipFile( - self.external_deps_path / f"{name}.zip", "r" - ) as zip_ref: - zip_ref.extractall(out_folder) - - print(f"Compiling {name}..") - src_folder = src_folder if src_folder else rev_name - c_filepath = self.external_deps_path / src_folder - try: - print(c_filepath) - - # Hacky patch fix until we can safely remove all this logic - if remove_line: - print("Patching Makefile..") - with open(c_filepath.joinpath("Makefile"), "r+") as f: - d = f.readlines() - f.seek(0) - for i in d: - if remove_line not in i: - f.write(i) - f.truncate() - - subprocess.check_call( - f"make {make_target}", - cwd=c_filepath, - shell=True, - ) - except subprocess.CalledProcessError as e: - raise RuntimeError(f"There was an error while compiling {name}") from e - - # Copy binary in a location which should be available on the PATH - # Note 1: if command_cls==install, self.install_scripts should correspond to /bin or ~/.local/bin - # Note 2: if command_cls==develop, self.install_scripts is None, so we are using a trick to get the value - # it would have with the `install` command - scripts_dir = self.install_scripts or get_setuptools_install_scripts_dir() - target_bin_path = Path(scripts_dir) / name - - print( - f"Copying binary from: " - f"{self.external_deps_path.joinpath(src_folder).joinpath(binary_name)}\n" - f"To: {target_bin_path}" - ) - shutil.copy( - self.external_deps_path.joinpath(src_folder).joinpath(binary_name), - target_bin_path, - ) - - def run(self): - if self.with_binaries: - self.external_deps_path = Path().cwd().joinpath("external_deps") - self.external_deps_path.mkdir(exist_ok=True) - - url = "https://www.civil.uwaterloo.ca/raven/files/" - self.install_binary_dep( - url, - "raven", - version=RAVEN_VERSION, - rev_name=f"RavenSource_v{RAVEN_VERSION}", - binary_name="Raven.exe", - remove_line="CXXFLAGS += -c++11", - ) - - # This works with python setup.py install, but produces this error with pip install: - # ERROR: ravenpy==0.1.0 did not indicate that it installed an .egg-info directory. - # Only setup.py projects generating .egg-info directories are supported. - # super().do_egg_install() - - # This works with pip install, but has the problem that it ignores install_requires - # when running with `python setup.py install`: - # https://stackoverflow.com/questions/21915469/python-setuptools-install-requires-is-ignored-when-overriding-cmdclass - command_cls.run(self) - - return InstallExternalDeps - - -setup( - author="David Huard", - author_email="huard.david@ouranos.ca", - python_requires=">=3.8", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python", - "Topic :: Scientific/Engineering :: Atmospheric Science", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Hydrology", - ], - description="A Python wrapper to setup and run the hydrologic modelling framework Raven.", - entry_points={ - "console_scripts": ["ravenpy=ravenpy.cli:main"], - }, - install_requires=requirements, - license="MIT license", - long_description=readme + "\n\n" + history, - long_description_content_type="text/x-rst", - include_package_data=True, - package_data={"ravenpy": ["*.csv", "*.zip"]}, - keywords="ravenpy", - name="ravenpy", - packages=find_packages( - include=[ - "ravenpy", - "ravenpy.*", - ], - ), - test_suite="tests", - tests_require=test_requirements, - extras_require=dict( - dev=dev_requirements, - docs=docs_requirements, - gis=gis_requirements, - ), - url="https://github.com/CSHS-CWRA/ravenpy", - version="0.11.0", - zip_safe=False, - cmdclass={ - "install": create_external_deps_install_class(install), - "develop": create_external_deps_install_class(develop), - }, -) From c34ccd94d0a1f096c491072111c3293d08755d3b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:48:23 -0400 Subject: [PATCH 08/67] note that clisops is mocked --- environment-rtd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-rtd.yml b/environment-rtd.yml index 45ab2102..e217980c 100644 --- a/environment-rtd.yml +++ b/environment-rtd.yml @@ -6,6 +6,7 @@ dependencies: - python >=3.8,<3.11 - autodoc-pydantic - click +# - clisops # mocked - ipykernel - ipython - jupyter_client From baa3385065ab97402c16e7b3902954086aeaf079 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:49:01 -0400 Subject: [PATCH 09/67] readability --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 595bf976..9d130409 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,9 @@ [tox] min_version = 4.0 -envlist = black, py{38,39,310}-numpy, docs +envlist = + black + py{38,39,310}-numpy + docs requires = pip >=21.0,<23.1 setuptools >=63.0,<65.6 From 46ff437aeddd4ad68c66628d0caa51b67697f38b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:58:58 -0400 Subject: [PATCH 10/67] fix CLI entrypoint --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ad939e0a..d84fb6fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,7 +145,7 @@ gis = [ ] [project.scripts] -raven = "ravenpy.cli:main" +ravenpy = "ravenpy.cli:main" [project.urls] "Homepage" = "https://ravenpy.readthedocs.io" From bb3bbd88ac82b0795f35dd6823a6405ee49995ea Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:24:45 -0400 Subject: [PATCH 11/67] update installation instructions and remove tox/Makefile calls to obsolete installation options --- Makefile | 8 +++++--- README.rst | 1 - docs/installation.rst | 45 +++++-------------------------------------- tox.ini | 2 -- 4 files changed, 10 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index ec0326f4..7a8d1994 100644 --- a/Makefile +++ b/Makefile @@ -89,9 +89,11 @@ release: dist ## package and upload a release twine upload dist/* dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel + flit build ls -l dist install: clean ## install the package to the active Python's site-packages - python setup.py install + python -m pip install --no-user . + +develop: clean ## install the package and development dependencies in editable mode to the active Python's site-packages + python -m pip install --no-user --editable ".[dev]" diff --git a/README.rst b/README.rst index 1bdd3368..70afd2f1 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,6 @@ Raven_ is an hydrological modeling framework that lets hydrologists build hydrol Note that version 0.20 includes major changes compared to the previous 0.12 release, and breaks backward compatibility. The benefits of these changes are a much more intuitive interface for configuring and running the model. - Features -------- diff --git a/docs/installation.rst b/docs/installation.rst index 7e59f63f..ac4d490b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -33,9 +33,9 @@ Custom Installation (Python/Pip) -------------------------------- .. warning:: - The following instructions will only work on POSIX-like systems (Unix/Linux; not supported on Windows). + As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future -If you wish to install RavenPy and its C-libraries manually, compiling the `Raven` binaries for your system, +If you wish to install RavenPy and its C-libraries manually, compiling the `Raven` binary for your system, you can install the entire system directly, placing them in the `bin` folder of your environment. In order to perform this from Ubuntu/Debian: @@ -48,51 +48,25 @@ Then, from your python environment, run: .. code-block:: console $ pip install ravenpy[gis] - $ pip install ravenpy[gis] --verbose --install-option="--with-binaries" - -.. warning:: - - It is imperative that the Python dependencies are pre-installed before running the `--with-binaries` - option; This install step will fail otherwise. If desired, the core functions of `RavenPy` can be installed without its GIS functionalities as well. This implementation of RavenPy is much lighter on dependencies and can be installed easily with `pip`, without the need for `conda` or `virtualenv`. -The only libraries required for RavenPy in this approach are a C++ compiler and the NetCDF4 development libraries. - -.. code-block:: console - - $ sudo apt-get install gcc libnetcdf-dev - .. code-block:: console $ pip install ravenpy - $ pip install ravenpy --verbose --install-option="--with-binaries" - -.. warning:: - - It is imperative that the Python dependencies are pre-installed before running the `--with-binaries` - option; This install step will fail otherwise. - -If for any reason you prefer to install without the binaries, from a fresh python environment, run the following: - -.. code-block:: console - - (ravenpy-env) $ pip install ravenpy[gis] But then you will be in charge of providing ``raven`` binaries on your PATH, or setting values for the ``RAVENPY_RAVEN_BINARY_PATH`` environment variable (as an absolute path) in the terminal/command prompt/shell used at runtime. -.. note:: - - The `virtualenv `_ implementation also works well, but the - GIS system libraries it depends on (specifically `GDAL` and `GEOS`) can be more difficult to configure. - Development Installation (from sources) --------------------------------------- +.. warning:: + As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future + The sources for RavenPy can be obtained from the GitHub repo: .. code-block:: console @@ -116,15 +90,6 @@ You can then install RavenPy with: # for the python dependencies (ravenpy) $ pip install --editable ".[dev]" -.. warning:: - - The following command will only work on POSIX-like systems (Unix/Linux; not supported on Windows). - -.. code-block:: console - - # for the Raven binaries - (ravenpy) $ pip install --editable "." --install-option="--with-binaries" - Install the pre-commit hook (to make sure that any code you contribute is properly formatted): .. code-block:: console diff --git a/tox.ini b/tox.ini index 9d130409..c7182f82 100644 --- a/tox.ini +++ b/tox.ini @@ -49,8 +49,6 @@ commands = python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" # Pin @ Date: Wed, 19 Apr 2023 17:27:09 -0400 Subject: [PATCH 12/67] remove check-manifest skip --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5360c61..4ae7db0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,5 +86,5 @@ ci: autoupdate_branch: '' autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' autoupdate_schedule: weekly - skip: [ check-manifest ] + skip: [ ] submodules: false From 79b482c51ee6afe04ad1b46bb7fec982c4620288 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:39:25 -0400 Subject: [PATCH 13/67] add pytest markers --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d84fb6fa..27c5643e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,3 +220,7 @@ python_files = "test_*.py" norecursedirs = ["src", ".git", "bin"] filterwarnings = ["ignore::UserWarning"] testpaths = "tests" +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "online: mark tests that require external services (deselect with '-m \"not online\"')" +] From 58b23d0d565e7d44e47cff5a1e8ea393c92f348b Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:39:34 -0400 Subject: [PATCH 14/67] add gdal to RtD build --- environment-rtd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-rtd.yml b/environment-rtd.yml index e217980c..c6f94c07 100644 --- a/environment-rtd.yml +++ b/environment-rtd.yml @@ -7,6 +7,7 @@ dependencies: - autodoc-pydantic - click # - clisops # mocked + - gdal - ipykernel - ipython - jupyter_client From 94c98649070aa8519a73921f16a325ccfb73be01 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:42:17 -0400 Subject: [PATCH 15/67] comment out tox GDAL compile --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c7182f82..e364ed4c 100644 --- a/tox.ini +++ b/tox.ini @@ -46,7 +46,7 @@ commands = # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.6.0 --no-binary netcdf4 # Deal with some GDAL silliness - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" + # python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" # Pin @ Date: Tue, 9 May 2023 17:31:18 -0400 Subject: [PATCH 16/67] update xclim, simplify build procedure --- environment.yml | 8 ++++---- pyproject.toml | 2 +- ravenpy/utilities/geoserver.py | 4 ++-- tests/conftest.py | 2 +- tests/test_nb_graphs.py | 2 +- tox.ini | 12 ++++-------- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/environment.yml b/environment.yml index 1a6f6b4e..eb89dcc2 100644 --- a/environment.yml +++ b/environment.yml @@ -20,9 +20,9 @@ dependencies: - lxml - matplotlib - netCDF4 - - numpy < 1.24 - - owslib >=0.24.1,<0.29 - - pandas < 2.0 + - numpy + - owslib >=0.29.1 + - pandas <2.0 - pint >=0.20 - pip - pre-commit @@ -39,6 +39,6 @@ dependencies: - spotpy - statsmodels - xarray <2022.11.0 # Pinned due to incompatibility with climpred @ 2.2.0 - - xclim >=0.40 + - xclim >=0.41 - xskillscore - wheel diff --git a/pyproject.toml b/pyproject.toml index 27c5643e..2f2789a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dependencies = [ "statsmodels", "wheel", "xarray<2022.11.0", # Pinned due to incompatibility with climpred @ 2.2.0 - "xclim>=0.40.0", + "xclim>=0.41.0", "xskillscore" ] diff --git a/ravenpy/utilities/geoserver.py b/ravenpy/utilities/geoserver.py index f70e8fc8..c1b949e8 100644 --- a/ravenpy/utilities/geoserver.py +++ b/ravenpy/utilities/geoserver.py @@ -52,11 +52,11 @@ # We store the contour of different hydrobasins domains hybas_dir = Path(__file__).parent.parent / "data" / "hydrobasins_domains" -hybas_pat = "hybas_lake_{}_lev01_v1c.zip" +hybas_pat = "hybas_lake_{domain}_lev01_v1c.zip" # This could be inferred from existing files in hybas_dir hybas_regions = ["na", "ar"] -hybas_domains = {dom: hybas_dir / hybas_pat.format(dom) for dom in hybas_regions} +hybas_domains = {dom: hybas_dir / hybas_pat.format(domain=dom) for dom in hybas_regions} def _get_location_wfs( diff --git a/tests/conftest.py b/tests/conftest.py index a3a82103..2c6b1e71 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest import xarray as xr from filelock import FileLock -from xclim.indicators.land import fit, stats +from xclim.indicators.generic import fit, stats from ravenpy.config import commands as rc from ravenpy.config.emulators import ( diff --git a/tests/test_nb_graphs.py b/tests/test_nb_graphs.py index a389d4cf..02bb7560 100644 --- a/tests/test_nb_graphs.py +++ b/tests/test_nb_graphs.py @@ -21,7 +21,7 @@ def test_spaghetti_annual_hydrograph(self, get_local_testdata): ) def test_ts_fit_graph(self, get_local_testdata): - from xclim.indicators.land import fit, stats + from xclim.indicators.generic import fit, stats ds = xr.open_dataset(get_local_testdata(self.hydrographs)) diff --git a/tox.ini b/tox.ini index e364ed4c..4638d8dc 100644 --- a/tox.ini +++ b/tox.ini @@ -2,10 +2,11 @@ min_version = 4.0 envlist = black - py{38,39,310}-numpy + py{38,39,310} docs requires = pip >=21.0,<23.1 + # Pin @ =63.0,<65.6 opts = --verbose @@ -38,17 +39,12 @@ install_command = python -m pip install --no-user {opts} {packages} deps = # numpy must be present in python env before GDAL is installed - !numpy: numpy + numpy + gdal=={env:GDAL_VERSION} commands = - # Install the latest NumPy and Numba before GDAL and netCDF4 are installed - numpy: python -m pip install --upgrade --force-reinstall --no-cache-dir numpy==1.23 numba # Install NetCDF4-Python via source files # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.6.0 --no-binary netcdf4 - # Deal with some GDAL silliness - # python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" - # Pin @ Date: Wed, 10 May 2023 12:25:42 -0400 Subject: [PATCH 17/67] Update pyproject.toml Co-authored-by: David Huard --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2f2789a1..51965644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,8 @@ build-backend = "flit_core.buildapi" [project] name = "ravenpy" authors = [ - {name = "David Huard", email = "huard.david@ouranos.ca"} + {name = "David Huard", email = "huard.david@ouranos.ca"}, + {name = "Richard Arsenault", email = "Richard.Arsenault@etsmtl.ca"} ] maintainers = [ {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"} From b6fcbba2168d8dd4a21d9e0c9b361aedfbfefc1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 16:26:02 +0000 Subject: [PATCH 18/67] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 51965644..9fa7fcf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "flit_core.buildapi" name = "ravenpy" authors = [ {name = "David Huard", email = "huard.david@ouranos.ca"}, - {name = "Richard Arsenault", email = "Richard.Arsenault@etsmtl.ca"} + {name = "Richard Arsenault", email = "Richard.Arsenault@etsmtl.ca"} ] maintainers = [ {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"} From 31175e10baf69b12c670fc40b3fc86a85dfd3cea Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 10 May 2023 18:18:13 -0400 Subject: [PATCH 19/67] WIP - add raven-hydro to tox build --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4638d8dc..04198793 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = py{38,39,310} docs requires = - pip >=21.0,<23.1 + pip >=21.0 # Pin @ =63.0,<65.6 opts = --verbose @@ -42,6 +42,8 @@ deps = numpy gdal=={env:GDAL_VERSION} commands = + # Experimental - Install raven-hydro via pip + python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Install NetCDF4-Python via source files # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.6.0 --no-binary netcdf4 From 48ba8adeadf97edf3cbcd983bc6a673175fd4075 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 14:03:35 -0400 Subject: [PATCH 20/67] try a workaround to not have ReadTheDocs run `--upgrade-strategy eager` --- .readthedocs.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index f7f63553..73f0f024 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,6 +17,8 @@ build: tools: python: "mambaforge-4.10" jobs: + post_install: + - pip install .[dev] pre_build: - sphinx-apidoc -o docs/apidoc --private --module-first ravenpy - sphinx-build -b linkcheck docs/ _build/linkcheck @@ -26,9 +28,9 @@ formats: all conda: environment: environment-rtd.yml -python: - install: - - method: pip - path: . - extra_requirements: - - dev +#python: +# install: +# - method: pip +# path: . +# extra_requirements: +# - dev From c314021e676619d83b31102b1f89b2f9d1a51082 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 15:05:55 -0400 Subject: [PATCH 21/67] dependency management --- environment.yml | 4 ++-- pyproject.toml | 4 ++-- tox.ini | 16 ++++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index eb89dcc2..27a1c74b 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - cftime - cf_xarray - click - - climpred >=2.1.0 + - climpred >=2.2.0 - dask - fiona >=1.9 - gdal >=3.1 @@ -38,7 +38,7 @@ dependencies: - shapely - spotpy - statsmodels - - xarray <2022.11.0 # Pinned due to incompatibility with climpred @ 2.2.0 + - xarray - xclim >=0.41 - xskillscore - wheel diff --git a/pyproject.toml b/pyproject.toml index 9fa7fcf8..7a682a84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "cftime", "cf-xarray", "click", - "climpred>=2.1", + "climpred>=2.2", "dask", "haversine", "matplotlib", @@ -54,7 +54,7 @@ dependencies = [ "spotpy", "statsmodels", "wheel", - "xarray<2022.11.0", # Pinned due to incompatibility with climpred @ 2.2.0 + "xarray", "xclim>=0.41.0", "xskillscore" ] diff --git a/tox.ini b/tox.ini index 04198793..4c45d5b2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ min_version = 4.0 envlist = black - py{38,39,310} + py{38,39,310}{-numpy,} docs requires = pip >=21.0 @@ -39,14 +39,18 @@ install_command = python -m pip install --no-user {opts} {packages} deps = # numpy must be present in python env before GDAL is installed - numpy - gdal=={env:GDAL_VERSION} + !numpy: numpy + !numpy: numba commands = - # Experimental - Install raven-hydro via pip - python -m pip install git+https://github.com/Ouranosinc/raven-hydro + # Install the latest NumPy and Numba before GDAL and netCDF4 are installed + numpy: python -m pip install --upgrade --force-reinstall --no-cache-dir numpy numba # Install NetCDF4-Python via source files # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.6.0 --no-binary netcdf4 + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 + # Deal with some GDAL silliness + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" + # Experimental - Install raven-hydro via pip + python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests pytest --cov # Coveralls requires access to a repo token set in .coveralls.yml in order to report stats From f3d59e6a91c0f2343cc04110aad3f0bef03c7912 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 15:29:14 -0400 Subject: [PATCH 22/67] GDAL --- tox.ini | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 4c45d5b2..afef175b 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ min_version = 4.0 envlist = black - py{38,39,310}{-numpy,} + py{38,39,310} docs requires = pip >=21.0 @@ -39,16 +39,15 @@ install_command = python -m pip install --no-user {opts} {packages} deps = # numpy must be present in python env before GDAL is installed - !numpy: numpy - !numpy: numba + numpy + gdal == {env:GDAL_VERSION} commands = - # Install the latest NumPy and Numba before GDAL and netCDF4 are installed - numpy: python -m pip install --upgrade --force-reinstall --no-cache-dir numpy numba # Install NetCDF4-Python via source files # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 # Deal with some GDAL silliness - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" + python -m pip install --force-reinstall setuptools<0.60 + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" --no-binary GDAL # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests From 6d32f07152e39b68c0c1dd393fc6a4474dbbb665 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 15:37:13 -0400 Subject: [PATCH 23/67] typo --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index afef175b..65a93326 100644 --- a/tox.ini +++ b/tox.ini @@ -46,7 +46,7 @@ commands = # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 # Deal with some GDAL silliness - python -m pip install --force-reinstall setuptools<0.60 + python -m pip install --force-reinstall setuptools<60.0 python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" --no-binary GDAL # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro From 3b19b0c5689344479c45b3a56f8bc2592701a381 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 15:55:24 -0400 Subject: [PATCH 24/67] fix deprecations --- tests/test_graphs.py | 2 +- tox.ini | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_graphs.py b/tests/test_graphs.py index c095f879..6b99e079 100644 --- a/tests/test_graphs.py +++ b/tests/test_graphs.py @@ -1,6 +1,6 @@ import numpy as np import xarray as xr -from xclim.indicators.land import fit, stats +from xclim.indicators.generic import fit, stats from ravenpy.utilities import graphs diff --git a/tox.ini b/tox.ini index 65a93326..24c9ed19 100644 --- a/tox.ini +++ b/tox.ini @@ -46,8 +46,7 @@ commands = # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 # Deal with some GDAL silliness - python -m pip install --force-reinstall setuptools<60.0 - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --global-option=build_ext --global-option="-I/usr/include/gdal" --no-binary GDAL + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --config--global-option=build_ext --global-option="-I/usr/include/gdal" --no-binary GDAL # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests From b2d14120bf99709073cc12598548ff2f38cfab68 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:04:28 -0400 Subject: [PATCH 25/67] typos --- tox.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 24c9ed19..6ca50300 100644 --- a/tox.ini +++ b/tox.ini @@ -42,11 +42,10 @@ deps = numpy gdal == {env:GDAL_VERSION} commands = - # Install NetCDF4-Python via source files - # Pin @v1.6.0 needed due to issue with PyPI wheels (see: https://github.com/Unidata/netcdf4-python/issues/1192) + # Regenerate NetCDF4-Python wheel from source files python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 - # Deal with some GDAL silliness - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --config--global-option=build_ext --global-option="-I/usr/include/gdal" --no-binary GDAL + # Rebuild GDAL with C bindings in order to gain access to _gdal_array objects + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --config-settings="--global-option=build_ext" --config-settings="--global-option=-I/usr/include/gdal" --no-binary GDAL # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests From c866eb5752c26144ae6ad2bb5a5e8000516e8ab9 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:04:33 -0400 Subject: [PATCH 26/67] add pooch --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7a682a84..77e3aa1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "holoviews", "hvplot", "isort", + "pooch", # Needed for cf_xarray>=0.8.1: https://github.com/xarray-contrib/cf-xarray/issues/440 "pre-commit", "pytest", "pytest-cov", From b83cdc459d322cc641e0300d9e1afd9dd4706401 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:27:44 -0400 Subject: [PATCH 27/67] simplify installation process further --- pyproject.toml | 3 +-- tox.ini | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 77e3aa1a..f8f25655 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ classifiers = [ dynamic = ["description", "version"] dependencies = [ "cftime", - "cf-xarray", + "cf-xarray[all]", "click", "climpred>=2.2", "dask", @@ -71,7 +71,6 @@ dev = [ "holoviews", "hvplot", "isort", - "pooch", # Needed for cf_xarray>=0.8.1: https://github.com/xarray-contrib/cf-xarray/issues/440 "pre-commit", "pytest", "pytest-cov", diff --git a/tox.ini b/tox.ini index 6ca50300..0330e8c9 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,7 @@ envlist = docs requires = pip >=21.0 - # Pin @ =63.0,<65.6 + setuptools >=63.0 opts = --verbose [testenv:black] @@ -44,8 +43,8 @@ deps = commands = # Regenerate NetCDF4-Python wheel from source files python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 - # Rebuild GDAL with C bindings in order to gain access to _gdal_array objects - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --config-settings="--global-option=build_ext" --config-settings="--global-option=-I/usr/include/gdal" --no-binary GDAL + # Rebuild GDAL in order to gain access to GDAL system-level objects + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --no-binary GDAL # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests @@ -54,6 +53,4 @@ commands = - coveralls --service=github allowlist_externals = make - git env - echo From 1fae833254a267f1adbfa98d1953e7d2efb0ccf8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:27:52 -0400 Subject: [PATCH 28/67] add a windows build --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1879a8e5..a523c7e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,10 @@ jobs: conda: name: Conda needs: black - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ['ubuntu-latest', 'windows-latest'] defaults: run: shell: bash -l {0} @@ -80,7 +83,6 @@ jobs: environment-file: environment.yml extra-specs: | mamba - pip<23.1 python=3.9 - name: Conda and Mamba versions run: | From db9795eec4407655a442e78afa0f24357a894e8f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:58:07 -0400 Subject: [PATCH 29/67] synchronize and simplify --- environment.yml | 5 ++--- pyproject.toml | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index 27a1c74b..c778c38d 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - netCDF4 - numpy - owslib >=0.29.1 - - pandas <2.0 + - pandas - pint >=0.20 - pip - pre-commit @@ -34,11 +34,10 @@ dependencies: - rioxarray - scikit-learn ==0.24.2 - scipy - - setuptools <=65.6 - shapely - spotpy - statsmodels - xarray - - xclim >=0.41 + - xclim >=0.43 - xskillscore - wheel diff --git a/pyproject.toml b/pyproject.toml index f8f25655..ff7505e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,9 @@ dependencies = [ "matplotlib", "netCDF4", "numpy", - "owslib<0.29", - "pandas<2.0", + "owslib>=0.29.1", + "pandas<2.0; python_version == '3.8'", + "pandas; python_version >= '3.9'", "pint>=0.20", "pydantic", "pymbolic", @@ -55,7 +56,7 @@ dependencies = [ "statsmodels", "wheel", "xarray", - "xclim>=0.41.0", + "xclim>=0.43.0", "xskillscore" ] From bc3a3b7325c2d89671c3690a618a50b8cff740ab Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 16:58:49 -0400 Subject: [PATCH 30/67] add a macOS pip build --- .github/workflows/main.yml | 45 ++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a523c7e2..d947be9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,45 +30,68 @@ jobs: run: tox -e black pip: - name: Pip with Python${{ matrix.python-version }} + name: Pip with Python${{ matrix.python-version }} (${{ matrix.os }}) needs: black - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: include: - tox-env: py38 python-version: "3.8" - - tox-env: py39-numpy + os: ubuntu-latest + - tox-env: py39 python-version: "3.9" + os: ubuntu-latest + - tox-env: py310 + python-version: "3.10" + os: ubuntu-latest - tox-env: py310 python-version: "3.10" + os: macos-latest steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install GDAL + + - name: Install GDAL (Ubuntu) + if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update sudo apt-get install libgdal-dev - - name: Install tox and setuptools + - name: Install GDAL (macOS) + if: matrix.os == 'macos-latest' + uses: tecolicom/actions-use-homebrew-tools@v1 + with: + tools: gdal + cache: yes + - name: Set GDAL_VERSION (Linux) + if: matrix.os == 'ubuntu-latest' + run: | + echo "GDAL_VERSION=$(gdal-config --version)" >> $GITHUB_ENV + - name: Set GDAL_VERSION (macOS) + if: matrix.os == 'macos-latest' + run: | + echo "GDAL_VERSION=$(gdalinfo --version | awk '{print $2}' | sed s'/.$//')" >> $GITHUB_ENV + + - name: Install tox run: | pip install tox~=4.0 - pip install --upgrade "setuptools<65.6" - name: Test with tox and report coverage - run: env GDAL_VERSION="$(gdal-config --version)" tox -e ${{ matrix.tox-env }} + run: | + python3 -m tox -e ${{ matrix.tox-env }} env: - LD_PRELOAD: /lib/x86_64-linux-gnu/libstdc++.so.6 # :"$LD_PRELOAD" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-${{ matrix.tox-env }} COVERALLS_PARALLEL: true conda: - name: Conda + name: Conda (${{ matrix.os }}) needs: black runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: ['ubuntu-latest', 'windows-latest'] defaults: @@ -76,6 +99,10 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@v3 + - name: Patch Environment File + if: matrix.os == 'windows-latest' + run: | + sed -i 's/climpred \>\=2.2.0/xesmf' environment.yml - name: Setup Conda (Micromamba) with Python3.9 uses: mamba-org/provision-with-micromamba@main with: From 1f64551408c058094e29d030a98e0b7502a21e11 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 17:03:32 -0400 Subject: [PATCH 31/67] fix sed call --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d947be9e..ecd93b62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,7 +102,7 @@ jobs: - name: Patch Environment File if: matrix.os == 'windows-latest' run: | - sed -i 's/climpred \>\=2.2.0/xesmf' environment.yml + sed -i 's/climpred >=2.2.0/xesmf/' environment.yml - name: Setup Conda (Micromamba) with Python3.9 uses: mamba-org/provision-with-micromamba@main with: From 8c6a500343e1fcf83641434b67bef02307582615 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 17:04:37 -0400 Subject: [PATCH 32/67] fix owslib version --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff7505e2..dcc2386f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,6 @@ dev = [ "geopandas>=0.9.0", "gdal", "lxml", - "owslib>=0.24.1,<0.29", "pyproj>=3.0.0", "rasterio", "rioxarray", From 5b4a99f017e18cf845a801cd5484027700429e77 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 17:24:38 -0400 Subject: [PATCH 33/67] fix owslib version --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index c778c38d..167a6865 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,7 @@ dependencies: - matplotlib - netCDF4 - numpy - - owslib >=0.29.1 + - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - pandas - pint >=0.20 - pip diff --git a/pyproject.toml b/pyproject.toml index dcc2386f..47810baa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "matplotlib", "netCDF4", "numpy", - "owslib>=0.29.1", + "owslib<0.29.0", # see: https://github.com/geopython/OWSLib/issues/871 "pandas<2.0; python_version == '3.8'", "pandas; python_version >= '3.9'", "pint>=0.20", From c1b2cd10ab2fa5d26bcb0493f0d9616ccef55add Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 17:24:49 -0400 Subject: [PATCH 34/67] tweaks --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ecd93b62..9bb106e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,6 +34,7 @@ jobs: needs: black runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: include: - tox-env: py38 @@ -122,7 +123,7 @@ jobs: conda list - name: Test RavenPy run: | - pytest --cov --numprocesses=logical + pytest --cov --numprocesses=0 - name: Report coverage run: coveralls --service=github env: From a3619f0a1c9f6e161d050ec41dcb0d1507cc686f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 12 May 2023 17:57:49 -0400 Subject: [PATCH 35/67] tweaks --- .github/workflows/main.yml | 10 +++++++--- tox.ini | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9bb106e9..cad226c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,7 +67,7 @@ jobs: with: tools: gdal cache: yes - - name: Set GDAL_VERSION (Linux) + - name: Set GDAL_VERSION (Ubuntu) if: matrix.os == 'ubuntu-latest' run: | echo "GDAL_VERSION=$(gdal-config --version)" >> $GITHUB_ENV @@ -94,7 +94,11 @@ jobs: strategy: fail-fast: false matrix: - os: ['ubuntu-latest', 'windows-latest'] + include: + - os: ubuntu-latest + num_processes: logical + - os: windows-latest + num_processes: 0 defaults: run: shell: bash -l {0} @@ -123,7 +127,7 @@ jobs: conda list - name: Test RavenPy run: | - pytest --cov --numprocesses=0 + pytest --cov --numprocesses=${{ matrix.num_processes }} - name: Report coverage run: coveralls --service=github env: diff --git a/tox.ini b/tox.ini index 0330e8c9..0c6ee3c6 100644 --- a/tox.ini +++ b/tox.ini @@ -41,8 +41,8 @@ deps = numpy gdal == {env:GDAL_VERSION} commands = - # Regenerate NetCDF4-Python wheel from source files - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4 --no-binary netcdf4 + # Regenerate NetCDF4-Python wheel from source files, use older version due to errors in new version (https://github.com/Unidata/netcdf4-python/issues/1192) + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.5.8 --no-binary netcdf4 # Rebuild GDAL in order to gain access to GDAL system-level objects python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --no-binary GDAL # Experimental - Install raven-hydro via pip From 21513fd4b105abbaf6936551a204cf4e81c41551 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 15 May 2023 17:00:22 -0400 Subject: [PATCH 36/67] update to newest geopandas API --- environment.yml | 2 +- pyproject.toml | 2 +- ravenpy/utilities/geoserver.py | 7 ++++--- tests/test_geoserver.py | 8 ++++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index 167a6865..12d13956 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - dask - fiona >=1.9 - gdal >=3.1 - - geopandas >=0.9 + - geopandas >=0.13.0 - haversine - holoviews - hvplot diff --git a/pyproject.toml b/pyproject.toml index 47810baa..ea24b784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ dev = [ # GIS Dependencies "affine", "fiona>=1.9", - "geopandas>=0.9.0", + "geopandas>=0.13.0", "gdal", "lxml", "pyproj>=3.0.0", diff --git a/ravenpy/utilities/geoserver.py b/ravenpy/utilities/geoserver.py index c1b949e8..7a4b3fdc 100644 --- a/ravenpy/utilities/geoserver.py +++ b/ravenpy/utilities/geoserver.py @@ -15,8 +15,8 @@ import inspect import json import os +import urllib.request import warnings -from io import BytesIO from pathlib import Path from typing import Iterable, Optional, Sequence, Tuple, Union from urllib.parse import urljoin @@ -362,10 +362,11 @@ def hydrobasins_upstream(feature: dict, domain: str) -> pd.DataFrame: # filter = PropertyIsEqualTo(propertyname=basin_family, literal=feature[basin_family]) # Fetch all features in the same basin - req = filter_hydrobasins_attributes_wfs( + request_url = filter_hydrobasins_attributes_wfs( attribute=basin_family, value=feature[basin_family], domain=domain ) - df = gpd.read_file(req) + with urllib.request.urlopen(url=request_url) as req: + df = gpd.read_file(filename=req, engine="pyogrio") # Filter upstream watersheds return _determine_upstream_ids( diff --git a/tests/test_geoserver.py b/tests/test_geoserver.py index 4a4456ca..867a0ec6 100644 --- a/tests/test_geoserver.py +++ b/tests/test_geoserver.py @@ -1,4 +1,5 @@ import tempfile +import urllib.request import numpy as np import pytest @@ -49,7 +50,8 @@ def test_get_hydrobasins_attributes_wfs(self, tmp_path): region_url = self.geoserver.filter_hydrobasins_attributes_wfs( attribute="MAIN_BAS", value=main_bas, domain="na" ) - gdf = self.gpd.read_file(region_url) + with urllib.request.urlopen(url=region_url) as req: + gdf = self.gpd.read_file(filename=req, engine="pyogrio") assert len(gdf) == 18 assert gdf.crs.to_epsg() == 4326 @@ -106,7 +108,9 @@ def test_get_hydro_routing_attributes_wfs(self): region_url = self.geoserver.filter_hydro_routing_attributes_wfs( attribute="IsLake", value="1.0", lakes="1km", level="07" ) - gdf = self.gpd.read_file(region_url) + with urllib.request.urlopen(url=region_url) as req: + gdf = self.gpd.read_file(filename=req, engine="pyogrio") + assert len(gdf) == 11415 @pytest.mark.slow From 8bdbcd8bebefaebcbc378a932b4a32726c17ca25 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 15 May 2023 17:14:16 -0400 Subject: [PATCH 37/67] add pyogrio --- environment.yml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 12d13956..58c03113 100644 --- a/environment.yml +++ b/environment.yml @@ -28,6 +28,7 @@ dependencies: - pre-commit - pydantic - pymbolic + - pyogrio - pyproj >=3.0 - rasterio - requests diff --git a/pyproject.toml b/pyproject.toml index ea24b784..1855af60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ dev = [ "geopandas>=0.13.0", "gdal", "lxml", + "pyogrio", "pyproj>=3.0.0", "rasterio", "rioxarray", From a7c3d4cb405e913f15dece7fd742acfea01d906f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 19 May 2023 12:44:07 -0400 Subject: [PATCH 38/67] remove lots of redundant dependencies, make mention to update installation steps once raven-hydro is ready, update install process in tox and readthedocs --- .readthedocs.yml | 4 ++-- docs/installation.rst | 11 ++++++++--- environment.yml | 2 +- pyproject.toml | 34 ++++------------------------------ tox.ini | 19 +++++++++++++------ 5 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 73f0f024..0afd6623 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -18,10 +18,10 @@ build: python: "mambaforge-4.10" jobs: post_install: - - pip install .[dev] + - pip install .[docs,gis] pre_build: - sphinx-apidoc -o docs/apidoc --private --module-first ravenpy - - sphinx-build -b linkcheck docs/ _build/linkcheck + - sphinx-build -b linkcheck docs/ _build/linkcheck || true formats: all diff --git a/docs/installation.rst b/docs/installation.rst index ac4d490b..08fb008a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,7 +6,8 @@ Full Installation (Anaconda) ---------------------------- For many reasons, we recommend using a `Conda environment `_ -to work with the full RavenPy installation. This implementation is able to manage the harder-to-install GIS dependencies,like `GDAL`. +to work with the full RavenPy installation. This implementation is able to manage the harder-to-install GIS dependencies, like `GDAL`. + Begin by creating an environment: .. code-block:: console @@ -32,6 +33,8 @@ as well as installs all the necessary Python and C libraries supporting GIS func Custom Installation (Python/Pip) -------------------------------- +# FIXME: Remove this warning and update docs once Ouranosinc/raven-hydro is finalized + .. warning:: As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future @@ -41,7 +44,7 @@ In order to perform this from Ubuntu/Debian: .. code-block:: console - $ sudo apt-get install gcc libnetcdf-dev gdal proj geos geopandas + $ sudo apt-get install gcc libnetcdf-dev gdal proj geos Then, from your python environment, run: @@ -64,6 +67,8 @@ terminal/command prompt/shell used at runtime. Development Installation (from sources) --------------------------------------- +# FIXME: Remove this warning and update docs once Ouranosinc/raven-hydro is finalized + .. warning:: As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future @@ -88,7 +93,7 @@ You can then install RavenPy with: .. code-block:: console # for the python dependencies - (ravenpy) $ pip install --editable ".[dev]" + (ravenpy) $ pip install --editable ".[dev,gis]" Install the pre-commit hook (to make sure that any code you contribute is properly formatted): diff --git a/environment.yml b/environment.yml index 58c03113..33fba91a 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - climpred >=2.2.0 - dask - fiona >=1.9 + - flit - gdal >=3.1 - geopandas >=0.13.0 - haversine @@ -41,4 +42,3 @@ dependencies: - xarray - xclim >=0.43 - xskillscore - - wheel diff --git a/pyproject.toml b/pyproject.toml index 1855af60..9bff84a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "matplotlib", "netCDF4", "numpy", - "owslib<0.29.0", # see: https://github.com/geopython/OWSLib/issues/871 + "owslib>=0.24.1,<0.29", # see: https://github.com/geopython/OWSLib/issues/871 "pandas<2.0; python_version == '3.8'", "pandas; python_version >= '3.9'", "pint>=0.20", @@ -54,7 +54,6 @@ dependencies = [ "scipy", "spotpy", "statsmodels", - "wheel", "xarray", "xclim>=0.43.0", "xskillscore" @@ -62,13 +61,13 @@ dependencies = [ [project.optional-dependencies] dev = [ - "Click", "black>=23.3.0", "bump2version", "coverage", "coveralls", "filelock", "flake8", + "flit", "holoviews", "hvplot", "isort", @@ -77,19 +76,7 @@ dev = [ "pytest-cov", "pytest-xdist>=3.2.0", "tox", - "watchdog", - "wheel", - # GIS Dependencies - "affine", - "fiona>=1.9", - "geopandas>=0.13.0", - "gdal", - "lxml", - "pyogrio", - "pyproj>=3.0.0", - "rasterio", - "rioxarray", - "shapely" + "watchdog" ] docs = [ "autodoc_pydantic", @@ -111,7 +98,6 @@ docs = [ "myst_nb", "nbsphinx", "numpydoc", - "owslib>=0.24.1,<0.29", "pandoc", "pymetalink", "s3fs", @@ -120,18 +106,7 @@ docs = [ "sphinx-codeautolink", "sphinx-copybutton", "sphinx-rtd-theme>=1.0", - "xesmf", - # GIS Dependencies - "affine", - "fiona>=1.9", - "geopandas>=0.9.0", - "gdal", - "lxml", - "owslib>=0.24.1,<0.29", - "pyproj>=3.0.0", - "rasterio", - "rioxarray", - "shapely" + "xesmf" ] gis = [ "affine", @@ -139,7 +114,6 @@ gis = [ "geopandas>=0.9.0", "gdal", "lxml", - "owslib>=0.24.1,<0.29", "pyproj>=3.0.0", "rasterio", "rioxarray", diff --git a/tox.ini b/tox.ini index 0c6ee3c6..06806cb6 100644 --- a/tox.ini +++ b/tox.ini @@ -14,13 +14,18 @@ skip_install = True deps = flake8 black -commands = make lint -allowlist_externals = make +commands = + make lint +allowlist_externals = + make [testenv:docs] -extras = docs -commands = make --directory=docs clean html -allowlist_externals = make +extras = + docs +commands = + make --directory=docs clean html +allowlist_externals = + make [testenv] setenv = @@ -32,7 +37,9 @@ passenv = GITHUB_* LD_PRELOAD RAVENPY_* -extras = dev +extras = + dev + gis download = true install_command = python -m pip install --no-user {opts} {packages} From 6d748aa1ea67b88b0518b2742021bad27ef1fe1b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 19 May 2023 12:57:11 -0400 Subject: [PATCH 39/67] update contributor documentation --- CONTRIBUTING.rst | 55 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c382f196..c683002d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -75,7 +75,7 @@ Ready to contribute? Here's how to set up `ravenpy` for local development. $ pre-commit install Special style and formatting checks will be run when you commit your changes. You - can always run the hooks on their own with: + can always run the hooks on their own with:: $ pre-commit run -a @@ -128,7 +128,7 @@ Tips To run a subset of tests:: -$ pytest tests.test_ravenpy + $ pytest tests.test_ravenpy Versioning/Tagging @@ -138,9 +138,9 @@ A reminder for the maintainers on how to deploy. Make sure all your changes are committed (including an entry in HISTORY.rst). Then run:: -$ bumpversion patch # possible: major / minor / patch -$ git push -$ git push --tags + $ bumpversion patch # possible: major / minor / patch + $ git push + $ git push --tags Packaging --------- @@ -148,21 +148,33 @@ Packaging When a new version has been minted (features have been successfully integrated test coverage and stability is adequate), maintainers should update the pip-installable package (wheel and source release) on PyPI as well as the binary on conda-forge. -The simple approach +The Automated Approach +~~~~~~~~~~~~~~~~~~~~~~ + +The simplest way to package `ravenpy` is to "publish" a version on GitHuh. GitHub CI Actions are presently configured to build the library and publish the packages on PyPI automatically. + +Tagged versions will trigger a GitHub Workflow (`tag-testpypi.yml`) that will attempt to build and publish the release on `TestPyPI `_. + +.. note:: + Should this step fail, changes may be needed in the package; Be sure to remove this tag on GitHub and locally, address any existing problems, and recreate the tag. + +To upload a new version to `PyPI `_, simply create a new "Published" release version on GitHub to trigger the upload workflow (`publish-pypi.yml`). When publishing on GitHub, the maintainer can either set the release notes manually (based on the `HISTORY.rst`), or set GitHub to generate release notes automatically. The choice of method is up to the maintainer. + +.. warning:: + A published version on TestPyPI/PyPI can never be overwritten. Be sure to verify that the package published at https://test.pypi.org/project/ravenpy/ matches expectations before publishing a release version on GitHub. + +The Manual Approach ~~~~~~~~~~~~~~~~~~~ -The simplest approach to packaging for general support (pip wheels) requires the following packages installed: - * setuptools - * wheel - * twine +The manual approach to library packaging for general support (pip wheels) requires that the `flit `_ library is installed. From the command line on your Linux distribution, simply run the following from the clone's main dev branch:: # To build the packages (sources and wheel) - $ python setup.py sdist bdist_wheel + $ flit build # To upload to PyPI - $ twine upload dist/* + $ flit publish The new version based off of the version checked out will now be available via `pip` (`$ pip install ravenpy`). @@ -177,8 +189,23 @@ In order to prepare an initial release on conda-forge, we *strongly* suggest con * https://github.com/conda-forge/staged-recipes Before updating the main conda-forge recipe, we echo the conda-forge documentation and *strongly* suggest performing the following checks: - * Ensure that dependencies and dependency versions correspond with those of the tagged version, with open or pinned versions for the `host` requirements. - * If possible, configure tests within the conda-forge build CI (e.g. `imports: ravenpy`, `commands: pytest ravenpy`) + * Ensure that dependencies and dependency versions correspond with those of the PyPI published version, with open or pinned versions for the `host` requirements. + * If possible, configure tests within the conda-forge build CI, e.g.: + +.. code-block:: yaml + + test: + source_files: + - tests + requires: + - pip + - pytest + - pytest-xdist + imports: + - ravenpy + commands: + - pip check + - pytest Subsequent releases ^^^^^^^^^^^^^^^^^^^ From 59c2bff95bc988f7381083ba60a0f18317112e45 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 19 May 2023 13:55:33 -0400 Subject: [PATCH 40/67] port remaining enhancements from experimental branch --- .github/workflows/main.yml | 5 +++-- environment.yml | 1 - mypy.ini | 19 ------------------- pyproject.toml | 21 +++++++++++++++++++++ ravenpy/utilities/geo.py | 10 ++++++---- tox.ini | 8 +++++--- 6 files changed, 35 insertions(+), 29 deletions(-) delete mode 100644 mypy.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cad226c5..7f557746 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,7 @@ jobs: run: tox -e black pip: - name: Pip with Python${{ matrix.python-version }} (${{ matrix.os }}) + name: Python${{ matrix.python-version }} (${{ matrix.os }}) needs: black runs-on: ${{ matrix.os }} strategy: @@ -88,7 +88,7 @@ jobs: COVERALLS_PARALLEL: true conda: - name: Conda (${{ matrix.os }}) + name: Python3.9 (${{ matrix.os }}) (Conda) needs: black runs-on: ${{ matrix.os }} strategy: @@ -112,6 +112,7 @@ jobs: uses: mamba-org/provision-with-micromamba@main with: cache-downloads: true + cache-env: true environment-file: environment.yml extra-specs: | mamba diff --git a/environment.yml b/environment.yml index 33fba91a..003d3b57 100644 --- a/environment.yml +++ b/environment.yml @@ -34,7 +34,6 @@ dependencies: - rasterio - requests - rioxarray - - scikit-learn ==0.24.2 - scipy - shapely - spotpy diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 153be1c2..00000000 --- a/mypy.ini +++ /dev/null @@ -1,19 +0,0 @@ -[mypy] -plugins = pydantic.mypy - -follow_imports = silent -warn_redundant_casts = True -warn_unused_ignores = True -disallow_any_generics = True -check_untyped_defs = True -no_implicit_reexport = False -ignore_missing_imports = True - -# for strict mypy: (this is the tricky one :-)) -disallow_untyped_defs = False - -[pydantic-mypy] -init_forbid_extra = True -init_typed = True -warn_required_dynamic_aliases = True -warn_untyped_fields = True diff --git a/pyproject.toml b/pyproject.toml index 9bff84a7..ce0b60c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dev = [ "holoviews", "hvplot", "isort", + "mypy", "pre-commit", "pytest", "pytest-cov", @@ -182,6 +183,26 @@ profile = "black" py_version = 38 append_only = true +[tool.mypy] +plugins = [ + "pydantic.mypy" +] +follow_imports = "silent" +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = false +ignore_missing_imports = true +# for strict mypy: (this is the tricky one :-)) +disallow_untyped_defs = false + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true +warn_untyped_fields = true + [tool.pytest.ini_options] addopts = [ "--verbose", diff --git a/ravenpy/utilities/geo.py b/ravenpy/utilities/geo.py index 9356bb67..95f2b05a 100644 --- a/ravenpy/utilities/geo.py +++ b/ravenpy/utilities/geo.py @@ -206,10 +206,10 @@ def generic_vector_reproject( Path to a file containing a valid vector layer. projected: Union[str, Path] Path to a file to be written. - source_crs : Union[str, dict, CRS] - Projection identifier (proj4) for the source geometry, Default: '+proj=longlat +datum=WGS84 +no_defs'. - target_crs : Union[str, dict, CRS] - Projection identifier (proj4) for the target geometry. + source_crs : Union[str, pyproj.crs.CRS] + CRS for the source geometry. Default: 4326. + target_crs : Union[str, pyproj.crs.CRS] + CRS for the target geometry. Returns ------- @@ -218,6 +218,8 @@ def generic_vector_reproject( if target_crs is None: raise ValueError("No target CRS is defined.") + if isinstance(target_crs, CRS): + target_crs = target_crs.to_dict() if isinstance(vector, Path): vector = vector.as_posix() diff --git a/tox.ini b/tox.ini index 06806cb6..e1234bb4 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,8 @@ envlist = requires = pip >=21.0 setuptools >=63.0 -opts = --verbose +opts = + -vv [testenv:black] skip_install = True @@ -48,10 +49,11 @@ deps = numpy gdal == {env:GDAL_VERSION} commands = - # Regenerate NetCDF4-Python wheel from source files, use older version due to errors in new version (https://github.com/Unidata/netcdf4-python/issues/1192) + # Regenerate NetCDF4-Python wheel from source files + # Use older version due to errors in new version (https://github.com/Unidata/netcdf4-python/issues/1192) python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.5.8 --no-binary netcdf4 # Rebuild GDAL in order to gain access to GDAL system-level objects - python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir GDAL=={env:GDAL_VERSION} --no-binary GDAL + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir gdal=={env:GDAL_VERSION} --no-binary gdal # Experimental - Install raven-hydro via pip python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests From be6fe55fc76cd13ec4c467f2b6d3b44a5dc5460f Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 10:42:54 -0400 Subject: [PATCH 41/67] update installation instructions --- docs/installation.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 08fb008a..36b40ddc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,8 +38,8 @@ Custom Installation (Python/Pip) .. warning:: As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future -If you wish to install RavenPy and its C-libraries manually, compiling the `Raven` binary for your system, -you can install the entire system directly, placing them in the `bin` folder of your environment. +If you wish to install RavenPy and its supporting system libraries manually, either compiling the `Raven` binary for your system or installing the pre-built binary, we encourage you to consult the `Raven` documentation (`Raven Downloads `_). + In order to perform this from Ubuntu/Debian: .. code-block:: console @@ -52,17 +52,13 @@ Then, from your python environment, run: $ pip install ravenpy[gis] -If desired, the core functions of `RavenPy` can be installed without its GIS functionalities as well. -This implementation of RavenPy is much lighter on dependencies and can be installed easily with `pip`, -without the need for `conda` or `virtualenv`. +If desired, the core functions of `RavenPy` can be installed without its GIS functionalities as well. This implementation of RavenPy is much lighter on dependencies and can be installed easily with `pip`, without the need for `conda` or `virtualenv`. .. code-block:: console $ pip install ravenpy -But then you will be in charge of providing ``raven`` binaries on your PATH, or setting values for the -``RAVENPY_RAVEN_BINARY_PATH`` environment variable (as an absolute path) in the -terminal/command prompt/shell used at runtime. +Once downloaded/compiled, ensure that the binary is placed in the `bin` folder of your environment and/or accessible on `$PATH` (Unix/Linux). The binary can also be pointed to manually (as an absolute path) by setting the environment variable ``RAVENPY_RAVEN_BINARY_PATH`` in the terminal/command prompt/shell used at runtime. Development Installation (from sources) --------------------------------------- From ebb5729f222b6b212df998f606a4a3af78091955 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 15:14:41 -0400 Subject: [PATCH 42/67] add raven-hydro to requirements --- README.rst | 14 +++++++------- pyproject.toml | 1 + tox.ini | 2 -- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 70afd2f1..1f385c05 100644 --- a/README.rst +++ b/README.rst @@ -15,12 +15,11 @@ Raven_ is an hydrological modeling framework that lets hydrologists build hydrol `RavenPy` provides a Python interface to Raven_, automating the creation of configuration files and allowing the model to be launched from Python. Results, or errors, are automatically parsed and exposed within the programming environment. This facilitates the launch of parallel simulations, multi-model prediction ensembles, sensitivity analyses and other experiments involving a large number of model runs. -Note that version 0.20 includes major changes compared to the previous 0.12 release, and breaks backward compatibility. The benefits of these changes are a much more intuitive interface for configuring and running the model. +Note that version 0.20.0 includes many major changes compared to the previous 0.12 release, and breaks backward compatibility, notably, the Raven model is now compiled/installed/managed via a dedicated pip/conda package `raven-hydro`_. The other benefits of these changes are a much more intuitive interface for configuring and running the model. Features -------- -* Download and compile Raven with `pip` * Configure, run and parse Raven outputs from Python * Utility command to create grid weight files * Extract physiographic information about watersheds @@ -39,13 +38,14 @@ RavenPy's development has been funded by CANARIE_ and Ouranos_ and would be not This package was created with Cookiecutter_ and the `Ouranosinc/cookiecutter-pypackage`_ project template. +.. _CANARIE: https://www.canarie.ca .. _Cookiecutter: https://github.com/audreyfeldroy/cookiecutter-pypackage +.. _Ouranos: https://www.ouranos.ca +.. _Ouranosinc/cookiecutter-pypackage: https://github.com/Ouranosinc/cookiecutter-pypackage .. _Raven: http://raven.uwaterloo.ca -.. _`CANARIE`: https://www.canarie.ca -.. _`Ouranos`: https://www.ouranos.ca -.. _`Ouranosinc/cookiecutter-pypackage`: https://github.com/Ouranosinc/cookiecutter-pypackage -.. _`docs`: https://www.civil.uwaterloo.ca/raven/files/v3.5/RavenManual_v3.5.pdf -.. _`installation docs`: https://ravenpy.readthedocs.io/en/latest/installation.html +.. _docs: https://www.civil.uwaterloo.ca/raven/files/v3.7/RavenManual_v3.7.pdf +.. _installation docs: https://ravenpy.readthedocs.io/en/latest/installation.html +.. _raven-hydro: https://github.com/Ouranosinc/raven-hydro .. |pypi| image:: https://img.shields.io/pypi/v/ravenpy.svg diff --git a/pyproject.toml b/pyproject.toml index ce0b60c9..19e6e6ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dependencies = [ "pint>=0.20", "pydantic", "pymbolic", + "raven-hydro==0.2.0", "requests", "scipy", "spotpy", diff --git a/tox.ini b/tox.ini index e1234bb4..fd98ca3c 100644 --- a/tox.ini +++ b/tox.ini @@ -54,8 +54,6 @@ commands = python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.5.8 --no-binary netcdf4 # Rebuild GDAL in order to gain access to GDAL system-level objects python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir gdal=={env:GDAL_VERSION} --no-binary gdal - # Experimental - Install raven-hydro via pip - python -m pip install git+https://github.com/Ouranosinc/raven-hydro # Run tests pytest --cov # Coveralls requires access to a repo token set in .coveralls.yml in order to report stats From 17fc9faaa33b550e56b504fd5a0b9e204b95557e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 15:52:15 -0400 Subject: [PATCH 43/67] add raven-hydro to requirements, pin pydantic to 1.10.8+ --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 19e6e6ae..2532fd35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "pandas<2.0; python_version == '3.8'", "pandas; python_version >= '3.9'", "pint>=0.20", - "pydantic", + "pydantic>=1.10.8", "pymbolic", "raven-hydro==0.2.0", "requests", @@ -116,6 +116,7 @@ gis = [ "geopandas>=0.9.0", "gdal", "lxml", + "pyogrio", "pyproj>=3.0.0", "rasterio", "rioxarray", From 6065713c6ebe065ae0f76ffe3cdfdc8c93bc3eb6 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 16:09:13 -0400 Subject: [PATCH 44/67] drop python3.9, install raven-hydro in conda by sdist --- .github/workflows/main.yml | 9 ++++++--- pyproject.toml | 3 +-- tox.ini | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f557746..f4fdba55 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,15 +37,15 @@ jobs: fail-fast: false matrix: include: - - tox-env: py38 - python-version: "3.8" - os: ubuntu-latest - tox-env: py39 python-version: "3.9" os: ubuntu-latest - tox-env: py310 python-version: "3.10" os: ubuntu-latest + - tox-env: py311 + python-version: "3.11" + os: ubuntu-latest - tox-env: py310 python-version: "3.10" os: macos-latest @@ -120,6 +120,9 @@ jobs: - name: Conda and Mamba versions run: | mamba --version + - name: Install raven-hydro (sdist) + run: | + pip install --upgrade --force-reinstall --no-deps --no-cache-dir raven-hydro=0.2.0 --no-binary raven-hydro - name: Install RavenPy run: | pip install -e ".[dev]" diff --git a/pyproject.toml b/pyproject.toml index 2532fd35..1fc95c64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,9 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python", "Topic :: Scientific/Engineering :: Atmospheric Science", "Topic :: Scientific/Engineering :: GIS", diff --git a/tox.ini b/tox.ini index fd98ca3c..1903ab6d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ min_version = 4.0 envlist = black - py{38,39,310} + py{39,310,311} docs requires = pip >=21.0 From c75901e45cbf2a3477375bb02317f98dd2fa119c Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 16:12:05 -0400 Subject: [PATCH 45/67] == --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f4fdba55..dc892ee2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,7 +122,7 @@ jobs: mamba --version - name: Install raven-hydro (sdist) run: | - pip install --upgrade --force-reinstall --no-deps --no-cache-dir raven-hydro=0.2.0 --no-binary raven-hydro + pip install --upgrade --force-reinstall --no-deps --no-cache-dir raven-hydro==0.2.0 --no-binary raven-hydro - name: Install RavenPy run: | pip install -e ".[dev]" From 2df1f850b56e6ee4583e1dbc9a3c4f24bc895154 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 23 May 2023 16:38:45 -0400 Subject: [PATCH 46/67] do not drop Python3.8, build raven-hydro from sources always --- .github/workflows/main.yml | 3 +++ pyproject.toml | 1 + tox.ini | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc892ee2..29946a66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,6 +37,9 @@ jobs: fail-fast: false matrix: include: + - tox-env: py38 + python-version: "3.8" + os: ubuntu-latest - tox-env: py39 python-version: "3.9" os: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 1fc95c64..42938896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/tox.ini b/tox.ini index 1903ab6d..20eb53e9 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ min_version = 4.0 envlist = black - py{39,310,311} + py{38,39,310,311} docs requires = pip >=21.0 @@ -54,6 +54,8 @@ commands = python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir netcdf4==1.5.8 --no-binary netcdf4 # Rebuild GDAL in order to gain access to GDAL system-level objects python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir gdal=={env:GDAL_VERSION} --no-binary gdal + # Build raven-hydro from source distribution + python -m pip install --upgrade --force-reinstall --no-deps --no-cache-dir raven-hydro==0.2.0 --no-binary raven-hydro # Run tests pytest --cov # Coveralls requires access to a repo token set in .coveralls.yml in order to report stats From f6c256ee310542434cc8175eca90476720a42d8d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 12:43:38 -0400 Subject: [PATCH 47/67] use raven-hydro conda package --- .github/workflows/main.yml | 3 --- environment.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29946a66..6b817a45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,9 +123,6 @@ jobs: - name: Conda and Mamba versions run: | mamba --version - - name: Install raven-hydro (sdist) - run: | - pip install --upgrade --force-reinstall --no-deps --no-cache-dir raven-hydro==0.2.0 --no-binary raven-hydro - name: Install RavenPy run: | pip install -e ".[dev]" diff --git a/environment.yml b/environment.yml index 003d3b57..e0490b1f 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - raven-hydro ==3.6 + - raven-hydro ==0.2.0 - python >=3.8 - affine - cftime From 6cc0c1d53ed4d579415c87ccf6fc71ebf91ba211 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 13:47:52 -0400 Subject: [PATCH 48/67] update CI --- .github/workflows/main.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b817a45..fd1266c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,7 +91,7 @@ jobs: COVERALLS_PARALLEL: true conda: - name: Python3.9 (${{ matrix.os }}) (Conda) + name: Python${{ matrix.python-version }} (${{ matrix.os }}) (Conda) needs: black runs-on: ${{ matrix.os }} strategy: @@ -100,8 +100,10 @@ jobs: include: - os: ubuntu-latest num_processes: logical + python-version: "3.9" - os: windows-latest num_processes: 0 + python-version: "3.9" defaults: run: shell: bash -l {0} @@ -111,18 +113,18 @@ jobs: if: matrix.os == 'windows-latest' run: | sed -i 's/climpred >=2.2.0/xesmf/' environment.yml - - name: Setup Conda (Micromamba) with Python3.9 - uses: mamba-org/provision-with-micromamba@main + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@v1 with: cache-downloads: true - cache-env: true + cache-environment: true environment-file: environment.yml - extra-specs: | - mamba - python=3.9 + create-args: >- + python=${{ matrix.python-version }} - name: Conda and Mamba versions run: | - mamba --version + conda --version + echo "micromamba: $(micromamba --version)" - name: Install RavenPy run: | pip install -e ".[dev]" From 4dad1fe968be89138669b6df93fdb171f2f92da9 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 14:02:15 -0400 Subject: [PATCH 49/67] install conda, force cache update --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd1266c0..03c86755 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,10 +116,11 @@ jobs: - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} uses: mamba-org/setup-micromamba@v1 with: - cache-downloads: true - cache-environment: true +# cache-downloads: true +# cache-environment: true environment-file: environment.yml create-args: >- + conda python=${{ matrix.python-version }} - name: Conda and Mamba versions run: | From 312ba8d63d74277dfbfb3da185da97149e966a7b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 14:14:22 -0400 Subject: [PATCH 50/67] try Python3.10 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03c86755..da22fdd7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,7 +100,7 @@ jobs: include: - os: ubuntu-latest num_processes: logical - python-version: "3.9" + python-version: "3.10" - os: windows-latest num_processes: 0 python-version: "3.9" From d72f71276c30dd59004a18a8251ae6f328647dcd Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 14:58:33 -0400 Subject: [PATCH 51/67] install climpred via PyPI --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index e0490b1f..c3c8698b 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - cftime - cf_xarray - click - - climpred >=2.2.0 +# - climpred >=2.2.0 - dask - fiona >=1.9 - flit @@ -25,7 +25,6 @@ dependencies: - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - pandas - pint >=0.20 - - pip - pre-commit - pydantic - pymbolic @@ -40,4 +39,5 @@ dependencies: - statsmodels - xarray - xclim >=0.43 + - xesmf - xskillscore From 4de60f569339e60359fc96be90ffef7d3ed317c5 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 15:41:27 -0400 Subject: [PATCH 52/67] Add a macos conda build --- .github/workflows/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da22fdd7..c973131d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,10 +100,13 @@ jobs: include: - os: ubuntu-latest num_processes: logical - python-version: "3.10" + python-version: "3.8" + - os: macos-latest + num_processes: logical + python-version: "3.9" - os: windows-latest num_processes: 0 - python-version: "3.9" + python-version: "3.10" defaults: run: shell: bash -l {0} From 870af309a228ea836d8daf7ede0578e27c6f7960 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 15:41:54 -0400 Subject: [PATCH 53/67] pin netcdf4 at or below 1.6.0 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c3c8698b..f92334ef 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - hvplot - lxml - matplotlib - - netCDF4 + - netcdf4 <=1.6.0 - numpy - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - pandas From 51bbc7f064b9a31c683d4ca1bf664c14bd3bcbb8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 15:46:38 -0400 Subject: [PATCH 54/67] pin netcdf4 below 1.6.3 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f92334ef..45338657 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - hvplot - lxml - matplotlib - - netcdf4 <=1.6.0 + - netcdf4 >=1.6.0,<1.6.3 - numpy - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - pandas From 78b470f4a35d193747d3614c0526b6cc60110938 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 15:49:25 -0400 Subject: [PATCH 55/67] unable to pin netcdf, constrain python --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 45338657..cf3d3e59 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - defaults dependencies: - raven-hydro ==0.2.0 - - python >=3.8 + - python >=3.8,<3.12 - affine - cftime - cf_xarray @@ -20,7 +20,7 @@ dependencies: - hvplot - lxml - matplotlib - - netcdf4 >=1.6.0,<1.6.3 + - netcdf4 - numpy - owslib <0.29.0 # see: https://github.com/geopython/OWSLib/issues/871 - pandas From ad5b88deb4360a1640ae57f4bb227daab9b594e8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 17:01:33 -0400 Subject: [PATCH 56/67] mark dap test as online --- tests/test_emulators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_emulators.py b/tests/test_emulators.py index c56d1291..1761d9dc 100644 --- a/tests/test_emulators.py +++ b/tests/test_emulators.py @@ -612,6 +612,7 @@ def test_routing_lievre_tutorial(get_local_testdata, tmp_path): assert out.hydrograph.q_sim[d].item() == pytest.approx(q_sim) +@pytest.mark.online def test_canopex(): CANOPEX_DAP = ( "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ets" From 575d854dac10aaf7b26e297f00931b70421ec2a1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 17:32:26 -0400 Subject: [PATCH 57/67] use a context manager to access netcdf data within a session-scope pytest fixture to prevent multiple file-access requests --- tests/conftest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2c6b1e71..889f96ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -529,13 +529,13 @@ def symbolic_config(salmon_meteo, salmon_hru, request): gextras = {"ALL": {"elevation": salmon_hru["land"]["elevation"]}} if name in ["HBVEC", "HYPR"]: - ds = xr.open_dataset(salmon_meteo) - gextras["ALL"]["monthly_ave_temperature"] = ( - ((ds.tmin + ds.tmax) / 2).groupby("time.month").mean().values.tolist() - ) - gextras["ALL"]["monthly_ave_evaporation"] = ( - ds.pet.groupby("time.month").mean().values.tolist() - ) + with xr.open_dataset(salmon_meteo) as ds: + gextras["ALL"]["monthly_ave_temperature"] = ( + ((ds.tmin + ds.tmax) / 2).groupby("time.month").mean().values.tolist() + ) + gextras["ALL"]["monthly_ave_evaporation"] = ( + ds.pet.groupby("time.month").mean().values.tolist() + ) # Extra attributes for emulator extras = {} From 7785bf5ca026ba5810178e8ec41f8c595e2c9d72 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 17:54:17 -0400 Subject: [PATCH 58/67] try context manager approach --- tests/test_nb_graphs.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/test_nb_graphs.py b/tests/test_nb_graphs.py index 02bb7560..10d31820 100644 --- a/tests/test_nb_graphs.py +++ b/tests/test_nb_graphs.py @@ -8,23 +8,22 @@ class TestNBGraphs: nbg = pytest.importorskip("ravenpy.utilities.nb_graphs") def test_hydrograph(self, get_local_testdata): - self.nbg.hydrographs(xr.open_dataset(get_local_testdata(self.hydrographs))) + with xr.open_dataset(get_local_testdata(self.hydrographs)) as ds: + self.nbg.hydrographs(ds) def test_mean_annual_hydrograph(self, get_local_testdata): - self.nbg.mean_annual_hydrograph( - xr.open_dataset(get_local_testdata(self.hydrographs)) - ) + with xr.open_dataset(get_local_testdata(self.hydrographs)) as ds: + self.nbg.mean_annual_hydrograph(ds) def test_spaghetti_annual_hydrograph(self, get_local_testdata): - self.nbg.spaghetti_annual_hydrograph( - xr.open_dataset(get_local_testdata(self.hydrographs)) - ) + with xr.open_dataset(get_local_testdata(self.hydrographs)) as ds: + self.nbg.spaghetti_annual_hydrograph(ds) def test_ts_fit_graph(self, get_local_testdata): from xclim.indicators.generic import fit, stats - ds = xr.open_dataset(get_local_testdata(self.hydrographs)) + with xr.open_dataset(get_local_testdata(self.hydrographs)) as ds: + ts = stats(ds.q_sim.load(), op="max", freq="M") - ts = stats(ds.q_sim, op="max", freq="M") params = fit(ts, dist="gamma") self.nbg.ts_fit_graph(ts, params) From b0806b5760137b9331c2799ab586963095b1b060 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 18:08:07 -0400 Subject: [PATCH 59/67] adjust test --- tests/test_geoserver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_geoserver.py b/tests/test_geoserver.py index 867a0ec6..f5dfa744 100644 --- a/tests/test_geoserver.py +++ b/tests/test_geoserver.py @@ -16,14 +16,14 @@ class TestHydroBASINS: gpd = pytest.importorskip("geopandas") sgeo = pytest.importorskip("shapely.geometry") - def test_select_hybas_na_domain(self): + def test_select_hybas_na_domain_bbox(self): bbox = (-68.0, 50.0) * 2 - dom = self.geoserver.select_hybas_domain(bbox) + dom = self.geoserver.select_hybas_domain(bbox=bbox) assert dom == "na" - def test_select_hybas_ar_domain(self): - bbox = (-114.65, 61.35) * 2 - dom = self.geoserver.select_hybas_domain(bbox) + def test_select_hybas_ar_domain_point(self): + point = -114.65, 61.35 + dom = self.geoserver.select_hybas_domain(point=point) assert dom == "ar" def test_get_hydrobasins_location_wfs(self, tmp_path): From d10039cb137c166f4788ac3abdee348288483a32 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 24 May 2023 18:26:37 -0400 Subject: [PATCH 60/67] adjust CI - drop windows --- .github/workflows/main.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c973131d..1e6a4a00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,13 +99,10 @@ jobs: matrix: include: - os: ubuntu-latest - num_processes: logical python-version: "3.8" - os: macos-latest - num_processes: logical python-version: "3.9" - - os: windows-latest - num_processes: 0 + - os: ubuntu-latest python-version: "3.10" defaults: run: @@ -119,8 +116,8 @@ jobs: - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} uses: mamba-org/setup-micromamba@v1 with: -# cache-downloads: true -# cache-environment: true + cache-downloads: true + cache-environment: true environment-file: environment.yml create-args: >- conda @@ -137,7 +134,7 @@ jobs: conda list - name: Test RavenPy run: | - pytest --cov --numprocesses=${{ matrix.num_processes }} + pytest --cov --numprocesses=logical - name: Report coverage run: coveralls --service=github env: From 9a84f582c4203f66aa342ec71be7f0fdd09a65cb Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 12:47:28 -0400 Subject: [PATCH 61/67] update CI for more build coverage --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e6a4a00..05f525b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,11 +99,13 @@ jobs: matrix: include: - os: ubuntu-latest - python-version: "3.8" - - os: macos-latest python-version: "3.9" - os: ubuntu-latest python-version: "3.10" + - os: ubuntu-latest + python-version: "3.11" + - os: macos-latest + python-version: "3.9" defaults: run: shell: bash -l {0} From 1617fb1198c29aeb3c4ef5b737aa2f70341676a8 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 13:08:53 -0400 Subject: [PATCH 62/67] update conda environment information --- environment-rtd.yml | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-rtd.yml b/environment-rtd.yml index c6f94c07..f3e050cf 100644 --- a/environment-rtd.yml +++ b/environment-rtd.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - python >=3.8,<3.11 + - python >=3.8,<3.12 - autodoc-pydantic - click # - clisops # mocked diff --git a/environment.yml b/environment.yml index cf3d3e59..521bd4ef 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - cftime - cf_xarray - click -# - climpred >=2.2.0 +# - climpred >=2.2.0 # conda package is incompatible with latest raven-hydro - dask - fiona >=1.9 - flit From 153e61d24419001b8716b3bb70d807d55022a68f Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 13:09:27 -0400 Subject: [PATCH 63/67] Update documentation to reflect changes --- CONTRIBUTING.rst | 2 +- HISTORY.rst | 10 ++++++++++ docs/installation.rst | 32 ++++++++++++++++++-------------- docs/usage.md | 10 +++++----- docs/user_api.rst | 7 ------- 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c683002d..693f0ede 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -119,7 +119,7 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 3.8, 3.9, and 3.10. Check +3. The pull request should work for Python 3.8, 3.9, 3.10, and 3.11. Check https://github.com/CSHS-CWRA/RavenPy/actions/workflows/main.yml and make sure that the tests pass for all supported Python versions. diff --git a/HISTORY.rst b/HISTORY.rst index ac110424..c545e75d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ History ======= +0.12.0 (2023-05-25) +------------------- + +* Ravenpy now employs a new method for installing the Raven model using the `raven-hydro `_ python package (based on `scikit-build-core`). +* Replaced `setup.py`, `requirements.txt`, and `Manifest.in` for `PEP 517 `_ compliance (`pyproject.toml`) using the flit backend. +* Dealt with an import-based error that occurred due to the sequence in which modules are loaded at import (attempting to call ravenpy before it is installed). +* Updated pre-commit hooks to include formatters and checkers for TOML files. +* The build recipes no longer build on each other, so when installing the dev or docs recipe, you must also install the gis recipe. +* Updated the GeoServer API calls to work with the GeoPandas v0.13.0. + 0.11.0 (2023-02-16) ------------------- diff --git a/docs/installation.rst b/docs/installation.rst index 36b40ddc..871e14d0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,7 +2,7 @@ Installation ============ -Full Installation (Anaconda) +Anaconda Python Installation ---------------------------- For many reasons, we recommend using a `Conda environment `_ @@ -26,19 +26,19 @@ RavenPy can then be installed directly via its `conda-forge` package by running: (ravenpy) $ conda install -c conda-forge ravenpy -This approach installs both the `Raven `_ binary directly to your environment `PATH`, +This approach installs the `Raven `_ binary directly to your environment `PATH`, as well as installs all the necessary Python and C libraries supporting GIS functionalities. +Python Installation (pip) +------------------------- -Custom Installation (Python/Pip) --------------------------------- - -# FIXME: Remove this warning and update docs once Ouranosinc/raven-hydro is finalized +.. warning:: + In order to compile the Raven model (provided by the `raven-hydro` package, a C++ compiler (`GCC`, `Clang`, `MSVC`, etc.) and either `GNU Make` (Linux/macOS) or `Ninja` (Windows) must be exposed on the `$PATH`. .. warning:: - As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future + The Raven model also requires that NetCDF4 libraries are installed on the system, exposed on the `$PATH`, and discoverable using the `FindNetCDF.cmake` helper script bundled with `raven-hydro`. -If you wish to install RavenPy and its supporting system libraries manually, either compiling the `Raven` binary for your system or installing the pre-built binary, we encourage you to consult the `Raven` documentation (`Raven Downloads `_). + On Linux, this can be provided by the `libnetcdf-dev` system library; On macOS by the `netcdf` homebrew package; And on Windows by using UNIDATA's [pre-built binaries](https://docs.unidata.ucar.edu/netcdf-c/current/winbin.html). In order to perform this from Ubuntu/Debian: @@ -58,15 +58,19 @@ If desired, the core functions of `RavenPy` can be installed without its GIS fun $ pip install ravenpy -Once downloaded/compiled, ensure that the binary is placed in the `bin` folder of your environment and/or accessible on `$PATH` (Unix/Linux). The binary can also be pointed to manually (as an absolute path) by setting the environment variable ``RAVENPY_RAVEN_BINARY_PATH`` in the terminal/command prompt/shell used at runtime. +Using A Custom Raven Model Binary +--------------------------------- -Development Installation (from sources) ---------------------------------------- +If you wish to install the `Raven` model, either compiling the `Raven` binary from sources for your system or installing the pre-built binary offered by UWaterloo, we encourage you to consult the `Raven` documentation (`Raven Downloads `_). -# FIXME: Remove this warning and update docs once Ouranosinc/raven-hydro is finalized +Once downloaded/compiled, the binary can be pointed to manually (as an absolute path) by setting the environment variable ``RAVENPY_RAVEN_BINARY_PATH`` in the terminal/command prompt/shell used at runtime. -.. warning:: - As of April 2023, this installation method does not install the `raven-hydro` hydrological model. You must install the `raven-hydro` model and add it to your `bin` or append it to your `$PATH` manually. This may change in the future +.. code-block:: console + + $ export RAVENPY_RAVEN_BINARY_PATH=/path/to/my/custom/raven + +Development Installation (from sources) +--------------------------------------- The sources for RavenPy can be obtained from the GitHub repo: diff --git a/docs/usage.md b/docs/usage.md index 427064fd..f4f2f186 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -12,7 +12,7 @@ In particular, RavenPy includes eight pre-configured model *emulators*. To run Raven using existing configuration files (`.rv*`), simply call the `run` function with the name of the configuration file and the path to the directory storing the RV files: -```python3 +```python from ravenpy import run output_path = run(modelname, configdir) @@ -25,7 +25,7 @@ output_path = run(modelname, configdir) The model outputs can be read with the `OutputReader` class: -```python3 +```python from ravenpy import OutputReader out = OutputReader(run_name, path=output_path) @@ -37,7 +37,7 @@ Note that this works only if simulated variables are stored as netCDF files, tha The class `EnsembleReader` does the same for an ensemble of model outputs, concatenating netCDF outputs along a new dimension: -```python3 +```python from ravenpy import EnsembleReader out = EnsembleReader( @@ -56,7 +56,7 @@ For more info, see {ref}`ensemble_reader`. Ravenpy comes packaged with pre-configured emulators, that is, Raven model configurations that can be modified on the fly. These emulators are made out of symbolic expressions, connecting model parameters to properties and coefficients. For example, the code below creates a model configuration for emulated model GR4JCN using the parameters given, as well as a `Gauge` configuration inferred by inspecting the `meteo.nc` file. -```python3 +```python from ravenpy.config.emulators import GR4JCN from ravenpy.config.commands import Gauge @@ -84,7 +84,7 @@ The RV files for the emulator above can be inspected using the `rvi`, `rvh`, `rv For convenience, `ravenpy` also proposes the `Emulator` class, designed to streamline the execution of the model and the retrieval of the results. -```python3 +```python from ravenpy import Emulator e = Emulator(config=gr4jcn, workdir="/tmp/gr4jcn/run_1") diff --git a/docs/user_api.rst b/docs/user_api.rst index 0efdc920..2ff77fb9 100644 --- a/docs/user_api.rst +++ b/docs/user_api.rst @@ -2,7 +2,6 @@ User API ======== - Execution ========= @@ -10,7 +9,6 @@ Execution :members: :noindex: - Configuration ============= @@ -22,8 +20,6 @@ Configuration :members: :noindex: - - Emulators ========= @@ -31,8 +27,6 @@ Emulators :members: :noindex: - - Extractors ========== @@ -44,7 +38,6 @@ Extractors :members: :noindex: - Utilities ========= From 75b51c907a0b5dced7aef58aacc17856dbf9b6e1 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 13:21:10 -0400 Subject: [PATCH 64/67] attempt to fix domain lookup bug --- ravenpy/utilities/geoserver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ravenpy/utilities/geoserver.py b/ravenpy/utilities/geoserver.py index 7a4b3fdc..8e239757 100644 --- a/ravenpy/utilities/geoserver.py +++ b/ravenpy/utilities/geoserver.py @@ -437,8 +437,9 @@ def select_hybas_domain( with open(fn, "rb") as f: zf = fiona.io.ZipMemoryFile(f) coll = zf.open(fn.stem + ".shp") - for _ in coll.filter(bbox=bbox): - return dom + for feat in coll.filter(bbox=bbox): + if isinstance(feat, fiona.Feature): + return dom raise LookupError(f"Could not find feature containing bbox: {bbox}.") From 698b7401f68568d9bf09f5650a924383ba276193 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 13:44:27 -0400 Subject: [PATCH 65/67] pin raven-hydro to v0.2.1 --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 521bd4ef..f5805807 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - raven-hydro ==0.2.0 + - raven-hydro ==0.2.1 - python >=3.8,<3.12 - affine - cftime diff --git a/pyproject.toml b/pyproject.toml index 42938896..bb94e084 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "pint>=0.20", "pydantic>=1.10.8", "pymbolic", - "raven-hydro==0.2.0", + "raven-hydro==0.2.1", "requests", "scipy", "spotpy", From 85b5c6468813d9374aa53271cf936fa8b2497d6f Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 14:10:21 -0400 Subject: [PATCH 66/67] allow errors for macOS --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 05f525b9..ff5140f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,12 +100,16 @@ jobs: include: - os: ubuntu-latest python-version: "3.9" + allow-errors: false - os: ubuntu-latest python-version: "3.10" + allow-errors: false - os: ubuntu-latest python-version: "3.11" + allow-errors: false - os: macos-latest python-version: "3.9" + allow-errors: true defaults: run: shell: bash -l {0} @@ -137,6 +141,7 @@ jobs: - name: Test RavenPy run: | pytest --cov --numprocesses=logical + continue-on-error: ${{ matrix.allow-errors }} - name: Report coverage run: coveralls --service=github env: From 870105359676c96cb12b6dfa83753ee368a8ce74 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Thu, 25 May 2023 14:46:12 -0400 Subject: [PATCH 67/67] allow errors for macOS on PyPI --- .github/workflows/main.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff5140f3..a153a574 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,18 +40,23 @@ jobs: - tox-env: py38 python-version: "3.8" os: ubuntu-latest + allow-errors: false - tox-env: py39 python-version: "3.9" os: ubuntu-latest + allow-errors: false - tox-env: py310 python-version: "3.10" os: ubuntu-latest + allow-errors: false - tox-env: py311 python-version: "3.11" os: ubuntu-latest + allow-errors: false - tox-env: py310 python-version: "3.10" os: macos-latest + allow-errors: true steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -85,6 +90,7 @@ jobs: - name: Test with tox and report coverage run: | python3 -m tox -e ${{ matrix.tox-env }} + continue-on-error: ${{ matrix.allow-errors }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-${{ matrix.tox-env }} @@ -100,16 +106,12 @@ jobs: include: - os: ubuntu-latest python-version: "3.9" - allow-errors: false - os: ubuntu-latest python-version: "3.10" - allow-errors: false - os: ubuntu-latest python-version: "3.11" - allow-errors: false - os: macos-latest python-version: "3.9" - allow-errors: true defaults: run: shell: bash -l {0} @@ -141,7 +143,6 @@ jobs: - name: Test RavenPy run: | pytest --cov --numprocesses=logical - continue-on-error: ${{ matrix.allow-errors }} - name: Report coverage run: coveralls --service=github env: