Skip to content

Commit

Permalink
Improve OSRFSourceCreation (#1008)
Browse files Browse the repository at this point in the history 
* Make static parameters to look like that

Use one option parameter to block jobs from changing names and repos and
use class members to support static members.

* Add an step to export TARBALL_NAME

* Support for adding downstream: releasepy + repo_uploader

* Added Global support to handle S3 paths

* Added _test_repository_uploader job

---------

Signed-off-by: Jose Luis Rivero <[email protected]>
  • Loading branch information
@j-rivero
j-rivero authored Sep 22, 2023
1 parent 59d8bbc commit 76e716f
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 18 deletions.
24 changes: 24 additions & 0 deletions jenkins-scripts/dsl/_configs_/Globals.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,4 +131,28 @@ class Globals
static String nontest_label(String original_label) {
return "(${original_label}) && !test-instance"
}

static String get_canonical_package_name(String package_name) {
return package_name.replaceAll('\\d*$', '')
}

static String _s3_releases_dir(String package_name) {
return get_canonical_package_name(package_name) + '/releases'
}

static String _s3_build_tarball_name(String package_name, String version) {
// canonical_name + version
return package_name.replaceAll('\\d*$', '') + '-' + version
}

static String s3_upload_tarball_path(String package_name) {
return 's3://osrf-distributions/' + _s3_releases_dir(package_name)
}

// Not yet in use. Requires changing release.py
static String s3_download_uri(String package_name, String version) {
return 'https://osrf-distributions.s3.amazonaws.com/' + \
_s3_releases_dir(package_name) + \
_s3_build_tarball_name(package_name, version)
}
}
106 changes: 89 additions & 17 deletions jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,43 +5,48 @@ import _configs_.Globals

class OSRFSourceCreation
{
static String properties_file = "package_name.prop"
static String package_name = ""

static void addParameters(Job job, Map default_params = [:])
{
package_name = default_params.find{ it.key == "PACKAGE"}?.value

job.with
{
parameters {
stringParam("PACKAGE_NAME",
default_params.find{ it.key == "PACKAGE_NAME"}?.value,
"Software name (i.e gz-cmake3)")
stringParam("SOURCE_REPO_URI",
default_params.find{ it.key == "SOURCE_REPO_URI"}?.value,
"GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)")
choiceParam('PACKAGE',
[default_params.find{ it.key == "PACKAGE"}?.value],
"Package name (can not be modified)")
choiceParam('SOURCE_REPO_URI',
[default_params.find{ it.key == "SOURCE_REPO_URI"}?.value],
"Software repository URL (can not be modified)")
stringParam("VERSION",
default_params.find{ it.key == "VERSION"}?.value,
"Packages version to be built or nightly (enable nightly build mode)")
stringParam("RELEASE_VERSION",
default_params.find{ it.key == "RELEASE_VERSION"}?.value,
"Packages release version")
"For downstream jobs: Packages release version")
stringParam("RELEASE_REPO_BRANCH",
default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value,
"Branch from the -release repo to be used")
"For downstream jobs: Branch from the -release repo to be used")
stringParam("UPLOAD_TO_REPO",
default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value,
"OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
"For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)")
}
}
}

static void create(Job job, Map default_params = [:])
static void create(Job job, Map default_params = [:], Map default_hidden_params = [:])
{
OSRFLinuxBuildPkgBase.create(job)
GenericRemoteToken.create(job)
OSRFSourceCreation.addParameters(job, default_params)

def pkg_sources_dir="pkgs"

job.with
{
label Globals.nontest_label("docker")

wrappers {
preBuildCleanup()
}
Expand All @@ -50,6 +55,9 @@ class OSRFSourceCreation
priority 100
}

def canonical_package_name = Globals.get_canonical_package_name(
default_params.find{ it.key == "PACKAGE"}.value)

steps {
systemGroovyCommand("""\
build.setDescription(
Expand All @@ -62,14 +70,78 @@ class OSRFSourceCreation
'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH'));
""".stripIndent()
)
shell("""\
#!/bin/bash -xe
# Use Jammy/amd64 as base image to generate sources
export DISTRO=jammy
export ARCH=amd64
/bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
""".stripIndent()
)
shell("""\
#!/bin/bash -xe
export DISTRO=jammy
export ARCH=amd64
#!/bin/bash -xe
# Export information from the build in properties_files. The tarball extraction helps to
# deal with changes in the compression of the tarballs.
tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \
-type f \
-name ${canonical_package_name}-\${VERSION}.tar.* \
-printf "%f\\n")
if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then
echo "Tarball name extraction returned \${tarball} which is not a one word string"
exit 1
fi
/bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash
""".stripIndent())
echo "TARBALL_NAME=\${tarball}" >> ${properties_file}
""".stripIndent()
)
}
}
}

// Useful to inject testing jobs
static void call_uploader_and_releasepy(Job job,
String repository_uploader_jobname,
String releasepy_jobname)
{
job.with
{
publishers {
postBuildScripts {
steps {
conditionalSteps {
condition {
not {
expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}')
}
}
steps {
// Invoke repository_uploader
downstreamParameterized {
trigger(repository_uploader_jobname) {
parameters {
currentBuild()
predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}',
S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)])
propertiesFile(properties_file) // TARBALL_NAME
}
}
}
downstreamParameterized {
trigger(releasepy_jobname) {
parameters {
currentBuild()
predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"])
propertiesFile(properties_file) // TARBALL_NAME
}
}
}
}
}
}
}
}
}
}
Expand Down
54 changes: 53 additions & 1 deletion jenkins-scripts/dsl/test.dsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,63 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job,
// releasing testing job
def releasepy_job = job("_test_releasepy")
OSRFReleasepy.create(releasepy_job, [DRY_RUN: true])
releasepy_job.with {
blockOn("_test_repository_uploader") {
blockLevel('GLOBAL')
scanQueueFor('ALL')
}
}
// gz source testing job
def gz_source_job = job("_test_gz_source")
OSRFSourceCreation.create(gz_source_job, [
PACKAGE_NAME: "gz-cmake3",
PACKAGE: "gz-cmake3" ,
SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"])
OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
'_test_repository_uploader',
'_test_releasepy')
// repository_uploader fake test job

def pkg_sources_dir = 'pkgs'
def repo_uploader = job("_test_repository_uploader")
OSRFBase.create(repo_uploader)
repo_uploader.with
{
label Globals.nontest_label("docker")

wrappers {
preBuildCleanup()
}

parameters
{
stringParam('PACKAGE','','Package name')
stringParam('TARBALL_NAME', '', 'Tarball name to upload')
stringParam('S3_UPLOAD_PATH','', 'S3 path to upload')
stringParam('UPLOAD_TO_REPO','none','repo to upload')
}

steps
{
copyArtifacts('_test_gz_source')
{
includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}")
buildSelector {
upstreamBuild()
}
}

shell("""\
#!/bin/bash -xe

# check that the tarball name actually exist

ls -R \${WORKSPACE}
test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME}

echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}"
""".stripIndent())
}
}

// -------------------------------------------------------------------
def outdated_job_runner = job("_test_outdated_job_runner")
Expand Down

0 comments on commit 76e716f

Please sign in to comment.