Regression tests for ODE solver and integration tests for production model is in place

.gitignore

@@ -154,6 +154,7 @@ playground/output
 # Ruff
 .ruff_cache
 .teamcity/.idea/
+.idea/
 
 # https://github.com/Deltares/Ribasim/issues/1652
 ribasim_qgis/tests/data/database.gpkg

.teamcity/Ribasim/Project.kt

@@ -7,11 +7,9 @@ import Ribasim.vcsRoots.Ribasim
 import Ribasim_Linux.RibasimLinuxProject
 import Ribasim_Windows.RibasimWindowsProject
 import Templates.*
+import Testbench.Testbench
 import jetbrains.buildServer.configs.kotlin.Project
-import jetbrains.buildServer.configs.kotlin.projectFeatures.activeStorage
-import jetbrains.buildServer.configs.kotlin.projectFeatures.awsConnection
-import jetbrains.buildServer.configs.kotlin.projectFeatures.githubIssues
-import jetbrains.buildServer.configs.kotlin.projectFeatures.s3Storage
+import jetbrains.buildServer.configs.kotlin.projectFeatures.*
 
 object Project : Project({
 
@@ -23,28 +21,39 @@ object Project : Project({
 
     template(GithubCommitStatusIntegration)
     template(GithubPullRequestsIntegration)
+    template(WindowsAgent)
+    template(BuildWindows)
+    template(LinuxAgent)
+    template(BuildLinux)
 
     features {
         awsConnection {
             id = "AmazonWebServicesAws"
             name = "Amazon Web Services (AWS)"
             regionName = "eu-west-3"
             credentialsType = static {
-                accessKeyId = "AKIAQBIN2MPWXSD2IZ5F"
-                secretAccessKey = "credentialsJSON:dba90026-9856-4f87-94d9-bab91f3f2d5c"
+                accessKeyId = "KwKRzscudy3GvRB8BN1Z"
+                secretAccessKey = "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2"
                 useSessionCredentials = false
-                stsEndpoint = "https://sts.eu-west-3.amazonaws.com"
+                stsEndpoint = "https://s3-console.deltares.nl"
             }
         }
-        activeStorage {
-            id = "PROJECT_EXT_106"
-            activeStorageID = "s3_ribasim"
-        }
         githubIssues {
             id = "PROJECT_EXT_107"
             displayName = "Ribasim GitHub Issues"
             repositoryURL = "https://github.com/Deltares/Ribasim"
         }
+        s3CompatibleStorage {
+            id = "PROJECT_EXT_171"
+            accessKeyID = "KwKRzscudy3GvRB8BN1Z"
+            accessKey = "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2"
+            endpoint = "https://s3.deltares.nl"
+            bucketName = "ribasim"
+        }
+        activeStorage {
+            id = "PROJECT_EXT_172"
+            activeStorageID = "PROJECT_EXT_171"
+        }
         s3Storage {
             id = "s3_ribasim"
             storageName = "s3"
@@ -61,4 +70,5 @@ object Project : Project({
 
     subProject(RibasimLinuxProject)
     subProject(RibasimWindowsProject)
+    subProject(Testbench)
 })

.teamcity/Ribasim_Linux/RibasimLinuxProject.kt

@@ -15,8 +15,6 @@ object RibasimLinuxProject : Project({
     buildType(Linux_BuildRibasim)
     buildType(Linux_TestRibasimBinaries)
 
-    template(LinuxAgent)
-    template(BuildLinux)
     template(TestBinariesLinux)
 })
 

.teamcity/Ribasim_Windows/RibasimWindowsProject.kt

@@ -17,8 +17,6 @@ object RibasimWindowsProject : Project({
     buildType(Windows_TestDelwaqCoupling)
     buildType(Windows_TestRibasimBinaries)
 
-    template(WindowsAgent)
-    template(BuildWindows)
     template(TestBinariesWindows)
     template(TestDelwaqCouplingWindows)
 })

.teamcity/Templates/IntegrationTest.kt

@@ -0,0 +1,75 @@
+package Templates
+
+import Ribasim_Windows.Windows_BuildRibasim
+import Ribasim_Linux.Linux_BuildRibasim
+import jetbrains.buildServer.configs.kotlin.Template
+import jetbrains.buildServer.configs.kotlin.buildFeatures.XmlReport
+import jetbrains.buildServer.configs.kotlin.buildFeatures.xmlReport
+import jetbrains.buildServer.configs.kotlin.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.triggers.schedule
+
+fun generateIntegrationTestHeader(platformOs: String): String {
+    if (platformOs == "Linux") {
+        return """
+                #!/bin/bash
+                # black magic
+                source /usr/share/Modules/init/bash
+
+                module load pixi
+            """.trimIndent() + System.lineSeparator()
+    }
+
+    return ""
+}
+
+open class IntegrationTest (platformOs: String) : Template() {
+
+    init {
+        name = "IntegrationTest_${platformOs}_Template"
+
+        artifactRules = """
+
+
+        """.trimIndent()
+
+        vcs {
+            root(Ribasim.vcsRoots.Ribasim, ". => ribasim")
+            cleanCheckout = true
+        }
+        params {
+            password("MiniO_credential_token", "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2")
+        }
+        val header = generateIntegrationTestHeader(platformOs)
+
+        steps {
+            script {
+                name = "Set up pixi"
+                id = "RUNNER_1505"
+                workingDir = "ribasim"
+                scriptContent = header +
+                """
+                pixi --version
+                pixi run install-ci
+                """.trimIndent()
+            }
+            script {
+                name = "Run integration tests"
+                id = "RUNNER_1507"
+                workingDir = "ribasim"
+                scriptContent = header +
+                """
+                pixi run python utils/get_benchmark.py %MiniO_credential_token% "hws_2024_7_0/"
+                pixi run model-integration-test
+                """.trimIndent()
+            }
+        }
+
+        failureConditions {
+            executionTimeoutMin = 90
+        }
+
+    }
+}
+
+object IntegrationTestWindows : IntegrationTest("Windows")
+object IntegrationTestLinux : IntegrationTest("Linux")

.teamcity/Templates/RegressionTest.kt

@@ -0,0 +1,75 @@
+package Templates
+
+import Ribasim_Windows.Windows_BuildRibasim
+import Ribasim_Linux.Linux_BuildRibasim
+import jetbrains.buildServer.configs.kotlin.Template
+import jetbrains.buildServer.configs.kotlin.buildFeatures.XmlReport
+import jetbrains.buildServer.configs.kotlin.buildFeatures.xmlReport
+import jetbrains.buildServer.configs.kotlin.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.triggers.schedule
+
+fun generateRegressionTestHeader(platformOs: String): String {
+    if (platformOs == "Linux") {
+        return """
+                #!/bin/bash
+                # black magic
+                source /usr/share/Modules/init/bash
+
+                module load pixi
+            """.trimIndent() + System.lineSeparator()
+    }
+
+    return ""
+}
+
+open class RegressionTest (platformOs: String) : Template() {
+
+    init {
+        name = "RegressionTest_${platformOs}_Template"
+
+        artifactRules = """
+
+
+        """.trimIndent()
+
+        vcs {
+            root(Ribasim.vcsRoots.Ribasim, ". => ribasim")
+            cleanCheckout = true
+        }
+        params {
+            password("MiniO_credential_token", "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2")
+        }
+        val header = generateRegressionTestHeader(platformOs)
+
+        steps {
+            script {
+                name = "Set up pixi"
+                id = "RUNNER_1509"
+                workingDir = "ribasim"
+                scriptContent = header +
+                        """
+                pixi --version
+                pixi run install-ci
+                """.trimIndent()
+            }
+            script {
+                name = "Run regression tests"
+                id = "RUNNER_1511"
+                workingDir = "ribasim"
+                scriptContent = header +
+                        """
+                pixi run python utils/get_benchmark.py %MiniO_credential_token% "benchmark/"
+                pixi run test-ribasim-regression
+                """.trimIndent()
+            }
+        }
+
+        failureConditions {
+            executionTimeoutMin = 30
+        }
+
+    }
+}
+
+object RegressionTestWindows : RegressionTest("Windows")
+object RegressionTestLinux : RegressionTest("Linux")

.teamcity/Testbench/IntegrationTestHWS.kt

@@ -0,0 +1,55 @@
+package Testbench.IntegrationTestHWS
+
+import Ribasim_Linux.Linux_BuildRibasim
+import Ribasim_Windows.Windows_BuildRibasim
+import Templates.*
+import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.triggers.schedule
+
+object IntegrationTestHWS : Project ({
+    id("IntegrationTestHWS")
+    name = "IntegrationTestHWS"
+
+    buildType(IntegrationTest_Windows)
+    buildType(IntegrationTest_Linux)
+
+    template(IntegrationTestWindows)
+    template(IntegrationTestLinux)
+})
+
+object IntegrationTest_Windows : BuildType({
+    name = "IntegrationTestWindows"
+    templates(WindowsAgent, GithubCommitStatusIntegration, IntegrationTestWindows)
+
+    triggers{
+        schedule {
+            id = ""
+            schedulingPolicy = daily {
+                hour = 0
+            }
+
+            branchFilter = "+:<default>"
+            triggerBuild = always()
+            withPendingChangesOnly = true
+        }
+    }
+})
+
+object IntegrationTest_Linux : BuildType({
+    name = "IntegrationTestLinux"
+    templates(LinuxAgent, GithubCommitStatusIntegration, IntegrationTestLinux)
+
+    triggers{
+        schedule {
+            id = ""
+            schedulingPolicy = daily {
+                hour = 0
+            }
+
+            branchFilter = "+:<default>"
+            triggerBuild = always()
+            withPendingChangesOnly = true
+        }
+    }
+})

.teamcity/Testbench/RegressionTestODESolve.kt

@@ -0,0 +1,57 @@
+package Testbench.RegressionTestODESolve
+
+import Ribasim_Windows.Windows_BuildRibasim
+import Ribasim_Linux.Linux_BuildRibasim
+import Templates.*
+import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.matrix
+import jetbrains.buildServer.configs.kotlin.triggers.schedule
+import jetbrains.buildServer.configs.kotlin.*
+
+object RegressionTestODESolve : Project({
+    id("RegressionTestODE")
+    name = "RegressionTestODE"
+
+    buildType(RegressionTest_Windows)
+    buildType(RegressionTest_Linux)
+
+    template(RegressionTestWindows)
+    template(RegressionTestLinux)
+})
+
+object RegressionTest_Windows : BuildType({
+    name = "RegressionTestWindows"
+    templates(WindowsAgent, GithubCommitStatusIntegration, RegressionTestWindows)
+
+    triggers{
+        schedule {
+            id = ""
+            schedulingPolicy = daily {
+                hour = 0
+            }
+
+            branchFilter = "+:<default>"
+            triggerBuild = always()
+            withPendingChangesOnly = true
+        }
+    }
+})
+
+object RegressionTest_Linux : BuildType({
+    name = "RegressionTestLinux"
+    templates(LinuxAgent, GithubCommitStatusIntegration, RegressionTestLinux)
+
+    triggers{
+        schedule {
+            id = ""
+            schedulingPolicy = daily {
+                hour = 0
+            }
+
+            branchFilter = "+:<default>"
+            triggerBuild = always()
+            withPendingChangesOnly = true
+        }
+    }
+})

.teamcity/Testbench/Testbench.kt

@@ -0,0 +1,12 @@
+package Testbench
+
+import Templates.*
+import Testbench.IntegrationTestHWS.IntegrationTestHWS
+import Testbench.RegressionTestODESolve.RegressionTestODESolve
+import jetbrains.buildServer.configs.kotlin.Project
+
+object Testbench : Project({
+    name = "Testbench"
+    subProject(IntegrationTestHWS)
+    subProject(RegressionTestODESolve)
+})