Merge branch 'main' into feat/delwaq-parse

.github/workflows/core_testmodels.yml

@@ -36,7 +36,7 @@ jobs:
           cache-registries: "true"
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Prepare pixi
         run: pixi run install-ci
       - name: Run testmodels with Ribasim Core

.github/workflows/core_tests.yml

@@ -36,7 +36,7 @@ jobs:
           cache-registries: "true"
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Test Ribasim Core
         run: |
           pixi run test-ribasim-core-cov

.github/workflows/docs.yml

@@ -27,7 +27,7 @@ jobs:
           cache-registries: "true"
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Prepare pixi
         run: pixi run install-ci
 

.github/workflows/julia_auto_update.yml

@@ -16,7 +16,7 @@ jobs:
           ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Update Julia manifest file
         run: |
           pixi run install-julia

.github/workflows/mypy.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Run mypy on python/ribasim
         run: |
           pixi run mypy-ribasim-python

.github/workflows/pixi_auto_update.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up pixi
         uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
           run-install: false
       - name: Update lockfiles
         run: |

.github/workflows/python_codegen.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Test if codegen runs without errors
         run: pixi run codegen
       - name: Ensure that no code has been generated

.github/workflows/python_tests.yml

@@ -29,7 +29,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: prefix-dev/[email protected]
         with:
-          pixi-version: v0.30.0
+          pixi-version: "latest"
       - name: Test Ribasim Python
         run: pixi run --environment ${{ matrix.pixi-environment }} test-ribasim-python-cov
       - name: Upload coverage to Codecov

.github/workflows/qgis.yml

@@ -19,7 +19,7 @@ jobs:
           - uses: actions/checkout@v4
           - uses: prefix-dev/[email protected]
             with:
-              pixi-version: v0.30.0
+              pixi-version: "latest"
           - name: Run tests
             run: pixi run test-ribasim-qgis-docker
           - name: Upload coverage to Codecov

Manifest.toml

@@ -1349,7 +1349,7 @@ version = "3.6.0"
 deps = ["Accessors", "Arrow", "BasicModelInterface", "CodecZstd", "ComponentArrays", "Configurations", "DBInterface", "DataInterpolations", "DataStructures", "Dates", "DiffEqBase", "DiffEqCallbacks", "EnumX", "FiniteDiff", "Graphs", "HiGHS", "IterTools", "JuMP", "Legolas", "LineSearches", "LinearSolve", "Logging", "LoggingExtras", "MetaGraphsNext", "OrdinaryDiffEqBDF", "OrdinaryDiffEqCore", "OrdinaryDiffEqLowOrderRK", "OrdinaryDiffEqNonlinearSolve", "OrdinaryDiffEqRosenbrock", "OrdinaryDiffEqSDIRK", "OrdinaryDiffEqTsit5", "PreallocationTools", "SQLite", "SciMLBase", "SparseArrays", "SparseConnectivityTracer", "StructArrays", "Tables", "TerminalLoggers", "TranscodingStreams"]
 path = "core"
 uuid = "aac5e3d9-0b8f-4d4f-8241-b1a7a9632635"
-version = "2024.10.0"
+version = "2024.11.0"
 
     [deps.Ribasim.extensions]
     RibasimMakieExt = ["Makie", "DataFrames"]

build/cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ribasim"
-version = "2024.10.0"
+version = "2024.11.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

core/Project.toml

@@ -2,7 +2,7 @@ name = "Ribasim"
 uuid = "aac5e3d9-0b8f-4d4f-8241-b1a7a9632635"
 authors = ["Deltares and contributors <[email protected]>"]
 manifest = "../Manifest.toml"
-version = "2024.10.0"
+version = "2024.11.0"
 
 [deps]
 Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"

core/integration_test/hws_integration_test.jl

@@ -17,13 +17,13 @@
 
     @testset "Results values" begin
         @test basin.node_id == basin_bench.node_id
-        @test all(q -> abs(q) < 0.06, basin.level - basin_bench.level)
+        @test all(q -> abs(q) < 0.2, basin.level - basin_bench.level)
     end
 
     timed = @timed Ribasim.run(toml_path)
 
-    # current benchmark is 600s
-    benchmark_runtime = 600
+    # current benchmark in seconds, TeamCity is up to 4x slower than local
+    benchmark_runtime = 32
     performance_diff =
         round((timed.time - benchmark_runtime) / benchmark_runtime * 100; digits = 2)
     if performance_diff < 0.0

core/regression_test/regression_test.jl

@@ -8,7 +8,7 @@
     config = Ribasim.Config(toml_path)
 
     solver_list =
-        ["QNDF", "Rosenbrock23", "TRBDF2", "Rodas5", "KenCarp4", "Tsit5", "ImplicitEuler"]
+        ["QNDF", "Rosenbrock23", "TRBDF2", "Rodas5P", "KenCarp4", "Tsit5", "ImplicitEuler"]
     sparse_on = [true, false]
     autodiff_on = [true, false]
 
@@ -38,13 +38,13 @@
                 # subgrid = Arrow.Table(subgrid_bytes)
 
                 @testset "Results values" begin
-                    @test basin.storage[1] ≈ 1.0
-                    @test basin.level[1] ≈ 0.044711584
-                    @test basin.storage[end] ≈ 16.530443267
+                    @test basin.storage[1] ≈ 1.0f0
+                    @test basin.level[1] ≈ 0.044711584f0
+                    @test basin.storage[end] ≈ 16.530443267f0
                     @test basin.level[end] ≈ 0.181817438
-                    @test flow.flow_rate[1] == basin.outflow_rate[1]
+                    @test flow.flow_rate[1] ≈ basin.outflow_rate[1]
                     @test all(q -> abs(q) < 1e-7, basin.balance_error)
-                    @test all(q -> abs(q) < 0.01, basin.relative_error)
+                    @test all(err -> abs(err) < 0.01, basin.relative_error)
                 end
             end
         end
@@ -105,7 +105,7 @@ end
                 @test all(q -> abs(q) < 1.0, basin.storage - basin_bench.storage)
                 @test all(q -> abs(q) < 0.5, basin.level - basin_bench.level)
                 @test all(q -> abs(q) < 1e-3, basin.balance_error)
-                @test all(q -> abs(q) < 2.5, basin.relative_error)
+                @test all(err -> abs(err) < 2.5, basin.relative_error)
             end
         end
     end
@@ -127,7 +127,7 @@ end
     flow_bench = Arrow.Table(flow_bytes_bench)
     basin_bench = Arrow.Table(basin_bytes_bench)
 
-    # TODO "Rosenbrock23" and "Rodas5" solver are resulting unsolvable gradients
+    # TODO "Rosenbrock23" and "Rodas5P" solver are resulting unsolvable gradients
     solver_list = ["QNDF"]
     sparse_on = [true, false]
     autodiff_on = [true, false]
@@ -167,7 +167,7 @@ end
                 @test basin.node_id == basin_bench.node_id
                 @test all(q -> abs(q) < 100.0, basin.storage - basin_bench.storage)
                 @test all(q -> abs(q) < 0.5, basin.level - basin_bench.level)
-                @test all(q -> abs(q) < 1e-3, basin.balance_error)
+                @test all(err -> abs(err) < 1e-3, basin.balance_error)
             end
         end
     end

core/src/bmi.jl

@@ -48,7 +48,7 @@ function BMI.get_value_ptr(model::Model, name::AbstractString)::AbstractVector{F
         p.subgrid.level
     elseif name == "user_demand.demand"
         vec(p.user_demand.demand)
-    elseif name == "user_demand.inflow"
+    elseif name == "user_demand.cumulative_inflow"
         u.user_demand_inflow
     else
         error("Unknown variable $name")

core/src/config.jl

@@ -100,8 +100,8 @@ const nodetypes = collect(keys(nodekinds))
     dtmin::Float64 = 0.0
     dtmax::Union{Float64, Nothing} = nothing
     force_dtmin::Bool = false
-    abstol::Float64 = 1e-6
-    reltol::Float64 = 1e-5
+    abstol::Float64 = 1e-7
+    reltol::Float64 = 1e-7
     water_balance_abstol::Float64 = 1e-3
     water_balance_reltol::Float64 = 1e-2
     maxiters::Int = 1e9

core/test/bmi_test.jl

@@ -11,11 +11,12 @@
     @test BMI.get_end_time(model) ≈ 3.16224e7
     BMI.update(model)
     @test BMI.get_current_time(model) ≈ dt0 atol = 5e-3
-    # cannot go back in time
-    @test_throws ErrorException BMI.update_until(model, dt0 / 2.0)
     @test BMI.get_current_time(model) ≈ dt0 atol = 5e-3
     BMI.update_until(model, 86400.0)
     @test BMI.get_current_time(model) == 86400.0
+    # cannot go back in time
+    @test_throws ErrorException BMI.update_until(model, 3600.0)
+    @test BMI.get_current_time(model) == 86400.0
 end
 
 @testitem "fixed timestepping" begin
@@ -67,7 +68,7 @@ end
         "basin.cumulative_drainage",
         "basin.subgrid_level",
         "user_demand.demand",
-        "user_demand.inflow",
+        "user_demand.cumulative_inflow",
     ]
         value_first = BMI.get_value_ptr(model, name)
         BMI.update_until(model, 86400.0)
@@ -86,7 +87,7 @@ end
     config = Ribasim.Config(toml_path; allocation_use_allocation = false)
     model = Ribasim.Model(config)
     demand = BMI.get_value_ptr(model, "user_demand.demand")
-    inflow = BMI.get_value_ptr(model, "user_demand.inflow")
+    inflow = BMI.get_value_ptr(model, "user_demand.cumulative_inflow")
     # One year in seconds
     year = model.integrator.p.user_demand.demand_itp[2][1].t[2]
     demand_start = 1e-3

core/test/control_test.jl

@@ -247,7 +247,7 @@ end
 
     t_switch = Ribasim.datetime_since(record.time[2], p.starttime)
     flow_table = DataFrame(Ribasim.flow_table(model))
-    @test all(filter(:time => time -> time <= t_switch, flow_table).flow_rate .> 0)
+    @test all(filter(:time => time -> time <= t_switch, flow_table).flow_rate .> -1e-12)
     @test all(
         isapprox.(
             filter(:time => time -> time > t_switch, flow_table).flow_rate,

core/test/data/config_test.toml

@@ -3,7 +3,7 @@ endtime = 2019-12-31
 crs = "EPSG:28992"
 input_dir = "../../generated_testmodels/lhm"
 results_dir = "../../generated_testmodels/lhm"
-ribasim_version = "2024.10.0"
+ribasim_version = "2024.11.0"
 
 [basin]
 time = "basin/time.arrow"

core/test/data/logging_test_loglevel_debug.toml

@@ -3,7 +3,7 @@ endtime = 2019-12-31
 crs = "EPSG:28992"
 input_dir = "."
 results_dir = "results"
-ribasim_version = "2024.10.0"
+ribasim_version = "2024.11.0"
 
 [logging]
 verbosity = "debug"

core/test/data/logging_test_no_loglevel.toml

@@ -3,4 +3,4 @@ endtime = 2019-12-31
 crs = "EPSG:28992"
 input_dir = "."
 results_dir = "results"
-ribasim_version = "2024.10.0"
+ribasim_version = "2024.11.0"

core/test/docs.toml

@@ -12,7 +12,7 @@ crs = "EPSG:4326"      # required
 input_dir = "."         # required
 results_dir = "results" # required
 
-ribasim_version = "2024.10.0" # required
+ribasim_version = "2024.11.0" # required
 
 # Specific tables can also go into Arrow files rather than the database.
 # For large tables this can benefit from better compressed file sizes.
@@ -31,8 +31,8 @@ dt = 60.0           # optional, remove for adaptive time stepping
 dtmin = 0.0         # optional, default 0.0
 dtmax = 0.0         # optional, default length of simulation
 force_dtmin = false # optional, default false
-abstol = 1e-6       # optional, default 1e-6
-reltol = 1e-5       # optional, default 1e-5
+abstol = 1e-7       # optional, default 1e-7
+reltol = 1e-7       # optional, default 1e-7
 water_balance_abstol = 1e-3 # optional, default 1e-3
 water_balance_reltol = 1e-2 # optional, default 1e-2
 maxiters = 1e9      # optional, default 1e9

docs/changelog.qmd

@@ -5,6 +5,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ## [Unreleased]
 
+## [v2024.11.0] - 2024-10-08
+
+This major new release contains many improvements.
+A new formulation allows much smaller water balance errors, which is combined with several performance improvements.
+Ribasim Python does more validation that was previously only done in the core.
+The Ribasim QGIS plugin now sets the relations between tables for easier model inspection.
+Adding `min_upstream_level` and `max_downstream_level` to Pump and Outlet means DiscreteControl is often no longer needed.
+The most significant breaking change is making the `node_id` and `edge_id` the index of the Node and Edge table; these need to be globally unique.
+
+Starting from this release Ribasim is labeled as beta software.
+Since development is currently mainly driven by applications in the Dutch water system, we expect that addition work needs to be done for general use outside the Netherlands.
+
+For coupled simulation with MODFLOW and/or MetaSWAP, this release is part of the [iMOD Coupler](https://deltares.github.io/iMOD-Documentation/coupler.html), specifically release [v2024.4.0](https://github.com/Deltares/imod_coupler/releases/tag/v2024.4.0)
+
 ### Added
 
 - Support discrete control based on an external concentration condition. [#1660](https://github.com/Deltares/Ribasim/pull/1660)
@@ -39,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 - Disallow missing priority parameter when using allocation. [#1745](https://github.com/Deltares/Ribasim/pull/1745)
 - Rename Outlet's `min_crest_level` to `min_upstream_level`. [#1788](https://github.com/Deltares/Ribasim/pull/1788)
 - Only allow flow under gravity in TabulatedRatingCurve. [#1795](https://github.com/Deltares/Ribasim/pull/1795)
+- Use `dtype_backend="pyarrow"` for Pandas DataFrames. [#1781](https://github.com/Deltares/Ribasim/pull/1781)
 
 ### Fixed
 - Remove oscillations in ManningResistance. [#1750](https://github.com/Deltares/Ribasim/pull/1750)

docs/concept/equations.qmd

@@ -157,8 +157,8 @@ For more a more in-depth discussion of numerical computations see [Numerical con
 
 There are many things that can influence the calculations times, for instance:
 
-- [Solver tolerance](https://diffeq.sciml.ai/stable/basics/faq/#What-does-tolerance-mean-and-how-much-error-should-I-expect):
-  By default we use absolute and relative tolerances of `1e-6` and `1e-5` respectively.
+- [Solver tolerance](https://docs.sciml.ai/DiffEqDocs/stable/basics/faq/#What-does-tolerance-mean-and-how-much-error-should-I-expect):
+  By default both the absolute and relative tolerance is `1e-7`.
 - [ODE solvers](https://diffeq.sciml.ai/stable/solvers/ode_solve/):
   The `QNDF` method we use is robust to oscillations and massive stiffness, however other solvers should be tried as well.
 - Forcing: Every time new forcing data is injected into the model, it needs to pause.

docs/dev/release.qmd

@@ -22,7 +22,7 @@ Before starting the release process, ensure that all tests are passing and that
 Determine the new version number like `2023.1.0`, filling in the current year, a bumped `MINOR` number for normal releases and a bumped `MICRO` number for non-breaking, hotfix releases.
 This follows `YYYY.MINOR.MICRO` from [calver](https://calver.org/).
 
-Create a branch that starts with `release`, like `release-2023-1.0`.
+Create a branch that starts with `release`, like `release-2023.1.0`.
 It needs to start with `release` to trigger extra TeamCity checks.
 
 Update the version numbers in the repository to the new version number.

pixi.lock

@@ -29760,7 +29760,7 @@ packages:
   timestamp: 1598024297745
 - kind: pypi
   name: ribasim
-  version: 2024.10.0
+  version: 2024.11.0
   path: python/ribasim
   sha256: c48692687129085ad19256cbf54c8df9853f6259b31997cc4a282fde86072751
   requires_dist:
@@ -29794,7 +29794,7 @@ packages:
   editable: true
 - kind: pypi
   name: ribasim-api
-  version: 2024.10.0
+  version: 2024.11.0
   path: python/ribasim_api
   sha256: dc882869854e0940ab3a12506f5b9508b04045793f6badb644579c43fa2b6cb9
   requires_dist:

pixi.toml

@@ -1,6 +1,6 @@
 [project]
 name = "Ribasim"
-version = "2024.10.0"
+version = "2024.11.0"
 description = "Water resources modeling"
 authors = ["Deltares and contributors <[email protected]>"]
 channels = ["conda-forge"]

python/ribasim/ribasim/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2024.10.0"
+__version__ = "2024.11.0"
 __schema_version__ = 2
 
 from ribasim.config import Allocation, Logging, Node, Solver

python/ribasim/ribasim/config.py

@@ -99,9 +99,9 @@ class Solver(ChildModel):
         If a smaller dt than dtmin is needed to meet the set error tolerances, the simulation stops, unless force_dtmin = true
         (Optional, defaults to False)
     abstol : float
-        The absolute tolerance for adaptive timestepping (Optional, defaults to 1e-6)
+        The absolute tolerance for adaptive timestepping (Optional, defaults to 1e-7)
     reltol : float
-        The relative tolerance for adaptive timestepping (Optional, defaults to 1e-5)
+        The relative tolerance for adaptive timestepping (Optional, defaults to 1e-7)
     maxiters : int
         The total number of linear iterations over the whole simulation. (Defaults to 1e9, only needs to be increased for extremely long simulations)
     sparse : bool