Refactor build pipeline (#1645)

This is a major PR reafctoring the teamcity pipelines.
The goal is:
- to avoid code duplication
- make it easy to extend (add or remove other build steps)
- make it more aligned with other projects (imod-python, coupler, etc)

The structure has changed slighlty
Both the windows and linux projects have a Main buildstep. This
buildstep is a composite buildstep. That means that is isn't building
anything directly, instead it triggers dependend buildsteps. This is the
starting point of the pipeline and only this builstep is directly
triggered from github

It then follows the chain and start the beginning.

Duplicate code have been extracted to Templates. The templates have been
made in such a way that they can both be used by the linux build aswell
as by the windows builds.

.teamcity/Ribasim/Project.kt

@@ -1,7 +1,12 @@
 package Ribasim
 
-import Ribasim.buildTypes.*
-import Ribasim.vcsRoots.*
+import Ribasim.buildTypes.GenerateTestmodels
+import Ribasim.buildTypes.Ribasim_MakeGitHubRelease
+import Ribasim.buildTypes.Ribasim_MakeQgisPlugin
+import Ribasim.vcsRoots.Ribasim
+import Ribasim_Linux.RibasimLinuxProject
+import Ribasim_Windows.RibasimWindowsProject
+import Templates.*
 import jetbrains.buildServer.configs.kotlin.Project
 import jetbrains.buildServer.configs.kotlin.projectFeatures.activeStorage
 import jetbrains.buildServer.configs.kotlin.projectFeatures.awsConnection
@@ -16,9 +21,8 @@ object Project : Project({
     buildType(Ribasim_MakeQgisPlugin)
     buildType(Ribasim_MakeGitHubRelease)
 
-    template(GitHubIntegrationTemplate)
-    template(Linux_1)
-    template(Windows_1)
+    template(GithubCommitStatusIntegration)
+    template(GithubPullRequestsIntegration)
 
     features {
         awsConnection {
@@ -55,6 +59,6 @@ object Project : Project({
         }
     }
 
-    subProject(Ribasim_Linux.Project)
-    subProject(Ribasim_Windows.Project)
+    subProject(RibasimLinuxProject)
+    subProject(RibasimWindowsProject)
 })

.teamcity/Ribasim/buildTypes/GenerateTestmodels.kt

@@ -1,18 +1,20 @@
 package Ribasim.buildTypes
 
+import Templates.GithubCommitStatusIntegration
 import jetbrains.buildServer.configs.kotlin.*
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.vcs
 
 object GenerateTestmodels : BuildType({
-    templates(GitHubIntegrationTemplate)
+    templates(GithubCommitStatusIntegration)
     name = "Generate Testmodels"
 
     artifactRules = """ribasim\generated_testmodels => generated_testmodels.zip"""
     publishArtifacts = PublishMode.SUCCESSFUL
 
     vcs {
         cleanCheckout = true
+        root(Ribasim.vcsRoots.Ribasim, ". => ribasim")
     }
 
     steps {

.teamcity/Ribasim/buildTypes/Ribasim_MakeGitHubRelease.kt

@@ -1,11 +1,16 @@
 package Ribasim.buildTypes
 
+import Ribasim_Linux.Linux_BuildRibasim
+import Ribasim_Linux.Linux_TestRibasimBinaries
+import Ribasim_Windows.Windows_BuildRibasim
+import Ribasim_Windows.Windows_TestRibasimBinaries
+import Templates.LinuxAgent
 import jetbrains.buildServer.configs.kotlin.*
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.vcs
 
 object Ribasim_MakeGitHubRelease : BuildType({
-    templates(Linux_1)
+    templates(LinuxAgent)
     name = "Make GitHub Release"
 
     params {
@@ -14,6 +19,7 @@ object Ribasim_MakeGitHubRelease : BuildType({
 
     vcs {
         root(Ribasim.vcsRoots.Ribasim)
+        cleanCheckout = true
     }
 
     steps {
@@ -52,7 +58,7 @@ object Ribasim_MakeGitHubRelease : BuildType({
                 artifactRules = "generated_testmodels.zip"
             }
         }
-        dependency(Ribasim_Linux.buildTypes.Linux_BuildRibasim) {
+        dependency(Linux_BuildRibasim) {
             snapshot {
                 onDependencyFailure = FailureAction.FAIL_TO_START
             }
@@ -62,7 +68,7 @@ object Ribasim_MakeGitHubRelease : BuildType({
                 artifactRules = "ribasim_linux.zip"
             }
         }
-        snapshot(Ribasim_Linux.buildTypes.Linux_TestRibasimBinaries) {
+        snapshot(Linux_TestRibasimBinaries) {
             onDependencyFailure = FailureAction.FAIL_TO_START
         }
         dependency(Ribasim_MakeQgisPlugin) {
@@ -75,7 +81,7 @@ object Ribasim_MakeGitHubRelease : BuildType({
                 artifactRules = "ribasim_qgis.zip"
             }
         }
-        dependency(Ribasim_Windows.buildTypes.Windows_BuildRibasim) {
+        dependency(Windows_BuildRibasim) {
             snapshot {
                 onDependencyFailure = FailureAction.FAIL_TO_START
             }
@@ -85,7 +91,7 @@ object Ribasim_MakeGitHubRelease : BuildType({
                 artifactRules = "ribasim_windows.zip"
             }
         }
-        snapshot(Ribasim_Windows.buildTypes.Windows_TestRibasimBinaries) {
+        snapshot(Windows_TestRibasimBinaries) {
             onDependencyFailure = FailureAction.FAIL_TO_START
         }
     }

.teamcity/Ribasim/buildTypes/Ribasim_MakeQgisPlugin.kt

@@ -1,16 +1,18 @@
 package Ribasim.buildTypes
 
+import Templates.LinuxAgent
 import jetbrains.buildServer.configs.kotlin.*
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 
 object Ribasim_MakeQgisPlugin : BuildType({
-    templates(Linux_1)
+    templates(LinuxAgent)
     name = "Make QGIS plugin"
 
     artifactRules = "ribasim_qgis.zip"
 
     vcs {
         root(Ribasim.vcsRoots.Ribasim)
+        cleanCheckout = true
     }
 
     steps {

.teamcity/Ribasim/vcsRoots/Ribasim.kt

@@ -7,9 +7,9 @@ object Ribasim : GitVcsRoot({
     url = "https://github.com/Deltares/Ribasim"
     branch = "main"
     branchSpec = """
-        +:refs/heads/*
+        +:refs/heads/main
         +:refs/tags/*
-        -:refs/heads/gh-pages
+        +:refs/heads/gh-readonly-queue/*
     """.trimIndent()
     useTagsAsBranches = true
     authMethod = password {

.teamcity/Ribasim_Linux/RibasimLinuxProject.kt

@@ -0,0 +1,78 @@
+package Ribasim_Linux
+
+import Ribasim.vcsRoots.Ribasim
+import Templates.*
+import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.FailureAction
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.triggers.vcs
+
+object RibasimLinuxProject : Project({
+    id("Ribasim_Linux")
+    name = "Ribasim_Linux"
+
+    buildType(Linux_Main)
+    buildType(Linux_BuildRibasim)
+    buildType(Linux_TestRibasimBinaries)
+
+    template(LinuxAgent)
+    template(BuildLinux)
+    template(TestBinariesLinux)
+})
+
+object Linux_Main : BuildType({
+    name = "RibasimMain"
+
+    templates(GithubPullRequestsIntegration)
+
+    allowExternalStatus = true
+    type = Type.COMPOSITE
+
+    vcs {
+        root(Ribasim, ". => ribasim")
+        cleanCheckout = true
+    }
+
+    triggers {
+        vcs {
+        }
+    }
+
+    dependencies {
+        snapshot(Linux_TestRibasimBinaries) {
+            onDependencyFailure = FailureAction.FAIL_TO_START
+        }
+    }
+})
+
+object Linux_BuildRibasim : BuildType({
+    templates(
+        LinuxAgent,
+        GithubCommitStatusIntegration,
+        BuildLinux
+    )
+
+    name = "Build Ribasim"
+
+    artifactRules = """ribasim\build\ribasim => ribasim_linux.zip"""
+})
+
+object Linux_TestRibasimBinaries : BuildType({
+    templates(LinuxAgent, GithubCommitStatusIntegration, TestBinariesLinux)
+    name = "Test Ribasim Binaries"
+
+    dependencies {
+        dependency(Linux_BuildRibasim) {
+            snapshot {
+            }
+
+            artifacts {
+                id = "ARTIFACT_DEPENDENCY_570"
+                cleanDestination = true
+                artifactRules = """
+                    ribasim_linux.zip!** => ribasim/build/ribasim
+                """.trimIndent()
+            }
+        }
+    }
+})