From 76e716f48d11ffc98a7298ad082c2e5578ba5f4d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Sep 2023 17:07:51 +0200 Subject: [PATCH] Improve OSRFSourceCreation (#1008) * Make static parameters to look like that Use one option parameter to block jobs from changing names and repos and use class members to support static members. * Add an step to export TARBALL_NAME * Support for adding downstream: releasepy + repo_uploader * Added Global support to handle S3 paths * Added _test_repository_uploader job --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 24 ++++ .../dsl/_configs_/OSRFSourceCreation.groovy | 106 +++++++++++++++--- jenkins-scripts/dsl/test.dsl | 54 ++++++++- 3 files changed, 166 insertions(+), 18 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 1386da8b6..2adc32511 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -131,4 +131,28 @@ class Globals static String nontest_label(String original_label) { return "(${original_label}) && !test-instance" } + + static String get_canonical_package_name(String package_name) { + return package_name.replaceAll('\\d*$', '') + } + + static String _s3_releases_dir(String package_name) { + return get_canonical_package_name(package_name) + '/releases' + } + + static String _s3_build_tarball_name(String package_name, String version) { + // canonical_name + version + return package_name.replaceAll('\\d*$', '') + '-' + version + } + + static String s3_upload_tarball_path(String package_name) { + return 's3://osrf-distributions/' + _s3_releases_dir(package_name) + } + + // Not yet in use. Requires changing release.py + static String s3_download_uri(String package_name, String version) { + return 'https://osrf-distributions.s3.amazonaws.com/' + \ + _s3_releases_dir(package_name) + \ + _s3_build_tarball_name(package_name, version) + } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index cb01aa186..a7eed1933 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -5,43 +5,48 @@ import _configs_.Globals class OSRFSourceCreation { + static String properties_file = "package_name.prop" + static String package_name = "" + static void addParameters(Job job, Map default_params = [:]) { + package_name = default_params.find{ it.key == "PACKAGE"}?.value + job.with { parameters { - stringParam("PACKAGE_NAME", - default_params.find{ it.key == "PACKAGE_NAME"}?.value, - "Software name (i.e gz-cmake3)") - stringParam("SOURCE_REPO_URI", - default_params.find{ it.key == "SOURCE_REPO_URI"}?.value, - "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)") + choiceParam('PACKAGE', + [default_params.find{ it.key == "PACKAGE"}?.value], + "Package name (can not be modified)") + choiceParam('SOURCE_REPO_URI', + [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value], + "Software repository URL (can not be modified)") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, - "Packages release version") + "For downstream jobs: Packages release version") stringParam("RELEASE_REPO_BRANCH", default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value, - "Branch from the -release repo to be used") + "For downstream jobs: Branch from the -release repo to be used") stringParam("UPLOAD_TO_REPO", default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, - "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") } } } - static void create(Job job, Map default_params = [:]) + static void create(Job job, Map default_params = [:], Map default_hidden_params = [:]) { OSRFLinuxBuildPkgBase.create(job) GenericRemoteToken.create(job) OSRFSourceCreation.addParameters(job, default_params) + def pkg_sources_dir="pkgs" + job.with { - label Globals.nontest_label("docker") - wrappers { preBuildCleanup() } @@ -50,6 +55,9 @@ class OSRFSourceCreation priority 100 } + def canonical_package_name = Globals.get_canonical_package_name( + default_params.find{ it.key == "PACKAGE"}.value) + steps { systemGroovyCommand("""\ build.setDescription( @@ -62,14 +70,78 @@ class OSRFSourceCreation 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); """.stripIndent() ) + shell("""\ + #!/bin/bash -xe + + # Use Jammy/amd64 as base image to generate sources + export DISTRO=jammy + export ARCH=amd64 + /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash + """.stripIndent() + ) shell("""\ - #!/bin/bash -xe - export DISTRO=jammy - export ARCH=amd64 + #!/bin/bash -xe + + # Export information from the build in properties_files. The tarball extraction helps to + # deal with changes in the compression of the tarballs. + tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \ + -type f \ + -name ${canonical_package_name}-\${VERSION}.tar.* \ + -printf "%f\\n") + if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then + echo "Tarball name extraction returned \${tarball} which is not a one word string" + exit 1 + fi - /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash - """.stripIndent()) + echo "TARBALL_NAME=\${tarball}" >> ${properties_file} + """.stripIndent() + ) + } + } + } + + // Useful to inject testing jobs + static void call_uploader_and_releasepy(Job job, + String repository_uploader_jobname, + String releasepy_jobname) + { + job.with + { + publishers { + postBuildScripts { + steps { + conditionalSteps { + condition { + not { + expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + } + } + steps { + // Invoke repository_uploader + downstreamParameterized { + trigger(repository_uploader_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', + S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + downstreamParameterized { + trigger(releasepy_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + } + } + } + } } } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index c8f542e8d..08c9a991a 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -20,11 +20,63 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, // releasing testing job def releasepy_job = job("_test_releasepy") OSRFReleasepy.create(releasepy_job, [DRY_RUN: true]) +releasepy_job.with { + blockOn("_test_repository_uploader") { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } +} // gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE_NAME: "gz-cmake3", + PACKAGE: "gz-cmake3" , SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"]) +OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, + '_test_repository_uploader', + '_test_releasepy') +// repository_uploader fake test job + +def pkg_sources_dir = 'pkgs' +def repo_uploader = job("_test_repository_uploader") +OSRFBase.create(repo_uploader) +repo_uploader.with +{ + label Globals.nontest_label("docker") + + wrappers { + preBuildCleanup() + } + + parameters + { + stringParam('PACKAGE','','Package name') + stringParam('TARBALL_NAME', '', 'Tarball name to upload') + stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') + stringParam('UPLOAD_TO_REPO','none','repo to upload') + } + + steps + { + copyArtifacts('_test_gz_source') + { + includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}") + buildSelector { + upstreamBuild() + } + } + + shell("""\ + #!/bin/bash -xe + + # check that the tarball name actually exist + + ls -R \${WORKSPACE} + test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME} + + echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}" + """.stripIndent()) + } +} // ------------------------------------------------------------------- def outdated_job_runner = job("_test_outdated_job_runner")