Improve OSRFSourceCreation

jenkins-scripts/dsl/_configs_/Globals.groovy

@@ -131,4 +131,28 @@ class Globals
    static String nontest_label(String original_label) {
     return "(${original_label}) && !test-instance"
    }
+
+   static String get_canonical_package_name(String package_name) {
+    return package_name.replaceAll('\\d*$', '')
+  }
+
+   static String _s3_releases_dir(String package_name) {
+    return get_canonical_package_name(package_name) + '/releases'
+   }
+
+   static String _s3_build_tarball_name(String package_name, String version) {
+    // canonical_name + version
+    return package_name.replaceAll('\\d*$', '') + '-' + version
+   }
+
+   static String s3_upload_tarball_path(String package_name) {
+    return 's3://osrf-distributions/' + _s3_releases_dir(package_name)
+   }
+
+   // Not yet in use. Requires changing release.py
+   static String s3_download_uri(String package_name, String version) {
+    return 'https://osrf-distributions.s3.amazonaws.com/' + \
+            _s3_releases_dir(package_name) + \
+            _s3_build_tarball_name(package_name, version)
+   }
 }

jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy

@@ -5,43 +5,48 @@ import _configs_.Globals
 
 class OSRFSourceCreation
 {
+  static String properties_file = "package_name.prop"
+  static String package_name = ""
+
   static void addParameters(Job job, Map default_params = [:])
   {
+    package_name = default_params.find{ it.key == "PACKAGE"}?.value
+
     job.with
     {
       parameters {
-        stringParam("PACKAGE_NAME",
-                    default_params.find{ it.key == "PACKAGE_NAME"}?.value,
-                    "Software name (i.e gz-cmake3)")
-        stringParam("SOURCE_REPO_URI",
-                    default_params.find{ it.key == "SOURCE_REPO_URI"}?.value,
-                    "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)")
+        choiceParam('PACKAGE',
+                    [default_params.find{ it.key == "PACKAGE"}?.value],
+                    "Package name (can not be modified)")
+        choiceParam('SOURCE_REPO_URI',
+                    [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value],
+                    "Software repository URL (can not be modified)")
         stringParam("VERSION",
                     default_params.find{ it.key == "VERSION"}?.value,
                     "Packages version to be built or nightly (enable nightly build mode)")
         stringParam("RELEASE_VERSION",
                     default_params.find{ it.key == "RELEASE_VERSION"}?.value,
-                    "Packages release version")
+                    "For downstream jobs: Packages release version")
         stringParam("RELEASE_REPO_BRANCH",
                     default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value,
-                    "Branch from the -release repo to be used")
+                    "For downstream jobs: Branch from the -release repo to be used")
         stringParam("UPLOAD_TO_REPO",
                     default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value,
-                    "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
+                    "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
       }
     }
   }
 
-  static void create(Job job, Map default_params = [:])
+  static void create(Job job, Map default_params = [:], Map default_hidden_params = [:])
   {
     OSRFLinuxBuildPkgBase.create(job)
     GenericRemoteToken.create(job)
     OSRFSourceCreation.addParameters(job, default_params)
 
+    def pkg_sources_dir="pkgs"
+
     job.with
     {
-      label Globals.nontest_label("docker")
-
       wrappers {
         preBuildCleanup()
       }
@@ -50,6 +55,9 @@ class OSRFSourceCreation
         priority 100
       }
 
+      def canonical_package_name = Globals.get_canonical_package_name(
+        default_params.find{ it.key == "PACKAGE"}.value)
+
       steps {
         systemGroovyCommand("""\
           build.setDescription(
@@ -62,14 +70,78 @@ class OSRFSourceCreation
           'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));
           """.stripIndent()
         )
+        shell("""\
+          #!/bin/bash -xe
+
+          # Use Jammy/amd64 as base image to generate sources
+          export DISTRO=jammy
+          export ARCH=amd64
 
+          /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
+          """.stripIndent()
+        )
         shell("""\
-            #!/bin/bash -xe
-            export DISTRO=jammy
-            export ARCH=amd64
+          #!/bin/bash -xe
+
+          # Export information from the build in properties_files. The tarball extraction helps to
+          # deal with changes in the compression of the tarballs.
+          tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \
+                       -type f \
+                       -name ${canonical_package_name}-\${VERSION}.tar.* \
+                       -printf "%f\\n")
+          if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then
+            echo "Tarball name extraction returned \${tarball} which is not a one word string"
+            exit 1
+          fi
 
-            /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
-            """.stripIndent())
+          echo "TARBALL_NAME=\${tarball}" >> ${properties_file}
+          """.stripIndent()
+        )
+      }
+    }
+  }
+
+  // Useful to inject testing jobs
+  static void call_uploader_and_releasepy(Job job,
+                                          String repository_uploader_jobname,
+                                          String releasepy_jobname)
+  {
+    job.with
+    {
+      publishers {
+        postBuildScripts {
+          steps {
+            conditionalSteps {
+             condition {
+              not {
+                expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}')
+                }
+              }
+              steps {
+                // Invoke repository_uploader
+                downstreamParameterized {
+                  trigger(repository_uploader_jobname) {
+                    parameters {
+                      currentBuild()
+                      predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}',
+                                       S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)])
+                      propertiesFile(properties_file) // TARBALL_NAME
+                    }
+                  }
+                }
+                downstreamParameterized {
+                  trigger(releasepy_jobname) {
+                    parameters {
+                      currentBuild()
+                      predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"])
+                      propertiesFile(properties_file) // TARBALL_NAME
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
       }
     }
   }

jenkins-scripts/dsl/test.dsl

@@ -20,11 +20,63 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job,
 // releasing testing job
 def releasepy_job = job("_test_releasepy")
 OSRFReleasepy.create(releasepy_job, [DRY_RUN: true])
+releasepy_job.with {
+      blockOn("_test_repository_uploader") {
+        blockLevel('GLOBAL')
+        scanQueueFor('ALL')
+      }
+}
 // gz source testing job
 def gz_source_job = job("_test_gz_source")
 OSRFSourceCreation.create(gz_source_job, [
-  PACKAGE_NAME: "gz-cmake3",
+  PACKAGE: "gz-cmake3" ,
   SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"])
+OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
+  '_test_repository_uploader',
+  '_test_releasepy')
+// repository_uploader fake test job
+
+def pkg_sources_dir = 'pkgs'
+def repo_uploader = job("_test_repository_uploader")
+OSRFBase.create(repo_uploader)
+repo_uploader.with
+{
+  label Globals.nontest_label("docker")
+
+  wrappers {
+      preBuildCleanup()
+  }
+
+  parameters
+  {
+    stringParam('PACKAGE','','Package name')
+    stringParam('TARBALL_NAME', '', 'Tarball name to upload')
+    stringParam('S3_UPLOAD_PATH','', 'S3 path to upload')
+    stringParam('UPLOAD_TO_REPO','none','repo to upload')
+  }
+
+  steps
+  {
+    copyArtifacts('_test_gz_source')
+    {
+      includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}")
+      buildSelector {
+        upstreamBuild()
+      }
+    }
+
+    shell("""\
+          #!/bin/bash -xe
+
+          # check that the tarball name actually exist
+
+          ls -R \${WORKSPACE}
+          test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME}
+
+          echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}"
+          """.stripIndent())
+  }
+}
 
 // -------------------------------------------------------------------
 def outdated_job_runner = job("_test_outdated_job_runner")