Ribasim Python: Add-API (#1110)

Fixes #760 Fixes #252 Follow ups: - More validation - Avoid blocking of database by qgis - Plotting of control edges - Reading of model --------- Co-authored-by: Martijn Visser <[email protected]> Co-authored-by: Hofer-Julian <[email protected]>
Deltares · Mar 12, 2024 · 006ff90 · 006ff90
1 parent 05fc42c
commit 006ff90
Show file tree

Hide file tree

Showing 56 changed files with 3,371 additions and 6,173 deletions.
diff --git a/.gitignore b/.gitignore
@@ -154,3 +154,4 @@ report.xml
 
 # Designated working dir for working on Ribasim models
 models/
+playground/output
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,7 +1,7 @@
 {
     "version": "0.2.0",
     "configurations": [
-         {
+        {
             "type": "julia",
             "request": "launch",
             "name": "Julia: current file",

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -2,7 +2,7 @@
     "[julia]": {
         "editor.formatOnSave": true
     },
-    "notebook.formatOnSave.enabled": true,
+    "notebook.formatOnSave.enabled": false,
     "notebook.codeActionsOnSave": {
         "source.fixAll.ruff": true,
         "source.organizeImports.ruff": true

diff --git a/core/src/graph.jl b/core/src/graph.jl
@@ -6,8 +6,10 @@ and data of edges (EdgeMetadata):
 [`EdgeMetadata`](@ref)
 """
 function create_graph(db::DB, config::Config, chunk_sizes::Vector{Int})::MetaGraph
-    node_rows =
-        execute(db, "SELECT node_id, node_type, subnetwork_id FROM Node ORDER BY fid")
+    node_rows = execute(
+        db,
+        "SELECT node_id, node_type, subnetwork_id FROM Node ORDER BY node_type, node_id",
+    )
     edge_rows = execute(
         db,
         "SELECT fid, from_node_type, from_node_id, to_node_type, to_node_id, edge_type, subnetwork_id FROM Edge ORDER BY fid",

diff --git a/core/src/read.jl b/core/src/read.jl
@@ -567,7 +567,7 @@ function DiscreteControl(db::DB, config::Config)::DiscreteControl
 
     return DiscreteControl(
         NodeID.(NodeType.DiscreteControl, condition.node_id), # Not unique
-        NodeID.(condition.listen_feature_type, condition.listen_feature_id),
+        NodeID.(condition.listen_node_type, condition.listen_node_id),
         condition.variable,
         look_ahead,
         condition.greater_than,
@@ -930,19 +930,11 @@ function Parameters(db::DB, config::Config)::Parameters
     return p
 end
 
-function get_nodetypes(db::DB)::Vector{String}
-    return only(execute(columntable, db, "SELECT node_type FROM Node ORDER BY fid"))
-end
-
 function get_ids(db::DB, nodetype)::Vector{Int}
-    sql = "SELECT node_id FROM Node WHERE node_type = $(esc_id(nodetype)) ORDER BY fid"
+    sql = "SELECT node_id FROM Node WHERE node_type = $(esc_id(nodetype)) ORDER BY node_id"
     return only(execute(columntable, db, sql))
 end
 
-function get_names(db::DB)::Vector{String}
-    return only(execute(columntable, db, "SELECT name FROM Node ORDER BY fid"))
-end
-
 function get_names(db::DB, nodetype)::Vector{String}
     sql = "SELECT name FROM Node where node_type = $(esc_id(nodetype)) ORDER BY fid"
     return only(execute(columntable, db, sql))

diff --git a/core/src/schema.jl b/core/src/schema.jl
@@ -183,8 +183,8 @@ end
 
 @version DiscreteControlConditionV1 begin
     node_id::Int
-    listen_feature_type::Union{Missing, String}
-    listen_feature_id::Int
+    listen_node_type::String
+    listen_node_id::Int
     variable::String
     greater_than::Float64
     look_ahead::Union{Missing, Float64}
@@ -199,7 +199,7 @@ end
 @version PidControlStaticV1 begin
     node_id::Int
     active::Union{Missing, Bool}
-    listen_node_type::Union{Missing, String}
+    listen_node_type::String
     listen_node_id::Int
     target::Float64
     proportional::Float64
@@ -210,7 +210,7 @@ end
 
 @version PidControlTimeV1 begin
     node_id::Int
-    listen_node_type::Union{Missing, String}
+    listen_node_type::String
     listen_node_id::Int
     time::DateTime
     target::Float64

diff --git a/core/src/util.jl b/core/src/util.jl
@@ -613,7 +613,11 @@ function allocation_path_exists_in_graph(
 end
 
 function has_main_network(allocation::Allocation)::Bool
-    return first(allocation.allocation_network_ids) == 1
+    if !is_active(allocation)
+        false
+    else
+        first(allocation.allocation_network_ids) == 1
+    end
 end
 
 function is_main_network(allocation_network_id::Int)::Bool

diff --git a/core/src/validation.jl b/core/src/validation.jl
@@ -91,7 +91,6 @@ function variable_nt(s::Any)
 end
 
 # functions used by sort(x; by)
-sort_by_fid(row) = row.fid
 sort_by_id(row) = row.node_id
 sort_by_time_id(row) = (row.time, row.node_id)
 sort_by_id_level(row) = (row.node_id, row.level)

diff --git a/core/test/run_models_test.jl b/core/test/run_models_test.jl
@@ -96,9 +96,9 @@
 
     @testset "Results values" begin
         @test flow.time[1] == DateTime(2020)
-        @test coalesce.(flow.edge_id[1:4], -1) == [-1, -1, 9, 11]
-        @test flow.from_node_id[1:4] == [6, typemax(Int), 0, 6]
-        @test flow.to_node_id[1:4] == [6, typemax(Int), typemax(Int), 0]
+        @test coalesce.(flow.edge_id[1:4], -1) == [-1, -1, 1, 2]
+        @test flow.from_node_id[1:4] == [6, 922, 6, 0]
+        @test flow.to_node_id[1:4] == [6, 922, 0, 922]
 
         @test basin.storage[1] ≈ 1.0
         @test basin.level[1] ≈ 0.044711584

diff --git a/core/test/validation_test.jl b/core/test/validation_test.jl
@@ -254,7 +254,7 @@ end
     @test logger.logs[3].kwargs[:control_state] == ""
     @test logger.logs[4].level == Error
     @test logger.logs[4].message == "Cannot connect a basin to a fractional_flow."
-    @test logger.logs[4].kwargs[:edge_id] == 6
+    @test logger.logs[4].kwargs[:edge_id] == 7
     @test logger.logs[4].kwargs[:id_src] == NodeID(:Basin, 2)
     @test logger.logs[4].kwargs[:id_dst] == NodeID(:FractionalFlow, 8)
 end
@@ -362,10 +362,10 @@ end
     @test length(logger.logs) == 2
     @test logger.logs[1].level == Error
     @test logger.logs[1].message ==
-          "Invalid edge type 'foo' for edge #0 from node #1 to node #2."
+          "Invalid edge type 'foo' for edge #1 from node #1 to node #2."
     @test logger.logs[2].level == Error
     @test logger.logs[2].message ==
-          "Invalid edge type 'bar' for edge #1 from node #2 to node #3."
+          "Invalid edge type 'bar' for edge #2 from node #2 to node #3."
 end
 
 @testitem "Subgrid validation" begin

diff --git a/docs/_quarto.yml b/docs/_quarto.yml
@@ -67,24 +67,23 @@ quartodoc:
       desc: The Model class represents an entire Ribasim model.
       contents:
         - Model
-    - title: Network
-      desc: The Node and Edge database layers define the network layout.
+    - title: Edge
+      desc: The Edge database layer.
       contents:
-        - Node
-        - Edge
+        - EdgeTable
     - title: Node types
       desc: Available node types to model different situations.
       contents:
-        - Basin
-        - FractionalFlow
-        - TabulatedRatingCurve
-        - Pump
-        - Outlet
-        - UserDemand
-        - LevelBoundary
-        - FlowBoundary
-        - LinearResistance
-        - ManningResistance
-        - Terminal
-        - DiscreteControl
-        - PidControl
+        - nodes.basin
+        - nodes.fractional_flow
+        - nodes.tabulated_rating_curve
+        - nodes.pump
+        - nodes.outlet
+        - nodes.user_demand
+        - nodes.level_boundary
+        - nodes.flow_boundary
+        - nodes.linear_resistance
+        - nodes.manning_resistance
+        - nodes.terminal
+        - nodes.discrete_control
+        - nodes.pid_control
diff --git a/docs/core/usage.qmd b/docs/core/usage.qmd
@@ -603,14 +603,14 @@ DiscreteControl is implemented based on [VectorContinuousCallback](https://docs.
 
 The condition schema defines conditions of the form 'the discrete_control node with this node id listens to whether the given variable of the node with the given listen feature id is grater than the given value'. If the condition variable comes from a time-series, a look ahead $\Delta t$ can be supplied.
 
-column              | type     | unit    | restriction
-------------------- | -------- | ------- | -----------
-node_id             | Int      | -       | sorted
-listen_feature_id   | Int      | -       | sorted per node_id
-listen_feature_type | String   | -       | known node type
-variable            | String   | -       | must be "level" or "flow_rate", sorted per listen_feature_id
-greater_than        | Float64  | various | sorted per variable
-look_ahead          | Float64  | $s$     | Only on transient boundary conditions, non-negative (optional, default 0)
+column            | type     | unit    | restriction
+----------------- | -------- | ------- | -----------
+node_id           | Int      | -       | sorted
+listen_node_type  | String   | -       | known node type
+listen_node_id    | Int      | -       | sorted per node_id
+variable          | String   | -       | must be "level" or "flow_rate", sorted per listen_node_id
+greater_than      | Float64  | various | sorted per variable
+look_ahead        | Float64  | $s$     | Only on transient boundary conditions, non-negative (optional, default 0)
 
 ## DiscreteControl / logic
 
@@ -642,16 +642,17 @@ The PidControl node controls the level in a basin by continuously controlling th
 
 In the future controlling the flow on a particular edge could be supported.
 
-column         | type     | unit     | restriction
--------------- | -------- | -------- | -----------
-node_id        | Int      | -        | sorted
-control_state  | String   | -        | (optional) sorted per node_id
-active         | Bool     | -        | (optional, default true)
-listen_node_id | Int      | -        | -
-target         | Float64  | $m$      | -
-proportional   | Float64  | $s^{-1}$ | -
-integral       | Float64  | $s^{-2}$ | -
-derivative     | Float64  | -        | -
+column           | type     | unit     | restriction
+---------------- | -------- | -------- | -----------
+node_id          | Int      | -        | sorted
+control_state    | String   | -        | (optional) sorted per node_id
+active           | Bool     | -        | (optional, default true)
+listen_node_type | Int      | -        | known node type
+listen_node_id   | Int      | -        | -
+target           | Float64  | $m$      | -
+proportional     | Float64  | $s^{-1}$ | -
+integral         | Float64  | $s^{-2}$ | -
+derivative       | Float64  | -        | -
 
 ## PidControl / time
 
@@ -663,15 +664,16 @@ these values interpolated linearly, and outside these values area constant given
 nearest time value.
 Note that a `node_id` can be either in this table or in the static one, but not both.
 
-column         | type     | unit     | restriction
--------------- | -------- | -------- | -----------
-node_id        | Int      | -        | sorted
-time           | DateTime | -        | sorted per node_id
-listen_node_id | Int      | -        | -
-target         | Float64  | $m$      | -
-proportional   | Float64  | $s^{-1}$ | -
-integral       | Float64  | $s^{-2}$ | -
-derivative     | Float64  | -        | -
+column           | type     | unit     | restriction
+---------------- | -------- | -------- | -----------
+node_id          | Int      | -        | sorted
+time             | DateTime | -        | sorted per node_id
+listen_node_type | Int      | -        | known node type
+listen_node_id   | Int      | -        | -
+target           | Float64  | $m$      | -
+proportional     | Float64  | $s^{-1}$ | -
+integral         | Float64  | $s^{-2}$ | -
+derivative       | Float64  | -        | -
 
 # Results