From 97762cd4b03511cabd826c4ac553039bc4197167 Mon Sep 17 00:00:00 2001 From: Mihai PLESA Date: Mon, 20 May 2019 10:06:41 +0100 Subject: [PATCH] multiple ci pipeline improvements (#2447) --- Jenkinsfile | 101 ++++++++++++++++++++----------------- script/rust_deps_config.py | 3 ++ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ef9cd2a7e3ab..55fae4c05ac1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { node { label "master" } } options { - disableConcurrentBuilds() + // TODO: set max. no. of concurrent builds to 2 timeout(time: 8, unit: "HOURS") timestamps() } @@ -33,35 +33,38 @@ pipeline { BRANCH_TO_BUILD = (env.CHANGE_BRANCH.equals(null) ? env.BRANCH_NAME : env.CHANGE_BRANCH) TARGET_BRANCH = (env.CHANGE_TARGET.equals(null) ? BRANCH_TO_BUILD : env.CHANGE_TARGET) BRANCH_EXISTS_IN_BB = httpRequest(url: GITHUB_API + "/brave-browser/branches/" + BRANCH_TO_BUILD, validResponseCodes: "100:499", authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).status.equals(200) - SKIP = false if (env.CHANGE_BRANCH) { - prNumber = readJSON(text: httpRequest(url: GITHUB_API + "/brave-core/pulls?head=brave:" + BRANCH_TO_BUILD, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content)[0].number - prDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-core/pulls/" + prNumber, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content) - SKIP = prDetails.mergeable_state.equals("draft") or prDetails.labels.count { label -> label.name.equals("CI/Skip") }.equals(1) + def prNumber = readJSON(text: httpRequest(url: GITHUB_API + "/brave-core/pulls?head=brave:" + BRANCH_TO_BUILD, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content)[0].number + def prDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-core/pulls/" + prNumber, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content) + env.SKIP = prDetails.mergeable_state.equals("draft") or prDetails.labels.count { label -> label.name.equals("CI/Skip") }.equals(1) } } } } - stage("continue") { - when { - expression { SKIP } - } + stage("abort") { steps { script { - print "PR is in draft or has \"CI/Skip\" label, aborting build!" - currentBuild.result = "ABORTED" + if (env.SKIP == true) { + print "Aborting build as PR is in draft or has \"CI/Skip\" label." + stopCurrentBuild() + } + for (build in getBuilds()) { + if (build.isBuilding() && build.getNumber() < env.BUILD_NUMBER.toInteger()) { + print "Aborting older running build " + build + build.doStop() + // build.finish(hudson.model.Result.ABORTED, new java.io.IOException("Aborting build")) + } + } } } } stage("checkout") { when { - expression { !SKIP } + expression { env.SKIP != true } } steps { sh """ - #!/bin/bash - - set +x + set -e if [ -d brave-browser ]; then rm -rf brave-browser @@ -79,15 +82,13 @@ pipeline { stage("branch-create") { when { allOf { - expression { !SKIP } + expression { env.SKIP != true } expression { !BRANCH_EXISTS_IN_BB } } } steps { sh """ - #!/bin/bash - - set +x + set -e cd brave-browser git checkout -b ${BRANCH_TO_BUILD} @@ -100,15 +101,13 @@ pipeline { stage("branch-rebase") { when { allOf { - expression { !SKIP } + expression { env.SKIP != true } expression { BRANCH_EXISTS_IN_BB } } } steps { sh """ - #!/bin/bash - - set +x + set -e cd brave-browser git checkout ${BRANCH_TO_BUILD} @@ -142,35 +141,47 @@ pipeline { """ } } - stage("sleep") { - when { - expression { !SKIP } - } - steps { - print "Sleeping 6m so new branch is discovered or associated PR created in brave-browser..." - sleep(time: 6, unit: "MINUTES") - } - } stage("build") { when { - expression { !SKIP } + expression { env.SKIP != true } } steps { script { - prDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-browser/pulls?head=brave:" + BRANCH_TO_BUILD, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content)[0] - prNumber = prDetails ? prDetails.number : "" - refToBuild = prNumber ? "PR-" + prNumber : URLEncoder.encode(BRANCH_TO_BUILD, "UTF-8") - params = [ - string(name: "CHANNEL", value: CHANNEL), - booleanParam(name: "WIPE_WORKSPACE", value: WIPE_WORKSPACE), - booleanParam(name: "RUN_INIT", value: RUN_INIT), - booleanParam(name: "DISABLE_SCCACHE", value: DISABLE_SCCACHE), - // TODO: add SKIP_SIGNING - booleanParam(name: "DEBUG", value: DEBUG) - ] - currentBuild.result = build(job: "brave-browser-build-pr/" + refToBuild, parameters: params, propagate: false).result + try { + startBraveBrowserBuild() + } + catch (hudson.AbortException ex) { + print "Sleeping 6m so new branch is discovered or associated PR created in brave-browser..." + sleep(time: 6, unit: "MINUTES") + startBraveBrowserBuild() + } } } } } } + +@NonCPS +def stopCurrentBuild() { + Jenkins.instance.getItemByFullName(env.JOB_NAME).getLastBuild().doStop() +} + +@NonCPS +def getBuilds() { + return Jenkins.instance.getItemByFullName(env.JOB_NAME).builds +} + +def startBraveBrowserBuild() { + def prDetails = readJSON(text: httpRequest(url: GITHUB_API + "/brave-browser/pulls?head=brave:" + BRANCH_TO_BUILD, authentication: GITHUB_CREDENTIAL_ID, quiet: !DEBUG).content)[0] + def prNumber = prDetails ? prDetails.number : "" + def refToBuild = prNumber ? "PR-" + prNumber : URLEncoder.encode(BRANCH_TO_BUILD, "UTF-8") + params = [ + string(name: "CHANNEL", value: CHANNEL), + booleanParam(name: "WIPE_WORKSPACE", value: WIPE_WORKSPACE), + booleanParam(name: "RUN_INIT", value: RUN_INIT), + booleanParam(name: "DISABLE_SCCACHE", value: DISABLE_SCCACHE), + // TODO: add SKIP_SIGNING + booleanParam(name: "DEBUG", value: DEBUG) + ] + currentBuild.result = build(job: "brave-browser-build-pr/" + refToBuild, parameters: params, propagate: false).result +} diff --git a/script/rust_deps_config.py b/script/rust_deps_config.py index f71ed3908e66..6f72bdbd4134 100755 --- a/script/rust_deps_config.py +++ b/script/rust_deps_config.py @@ -4,5 +4,8 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. # Version number and URL for pre-configured rust dependency package, e.g. rust_deps_mac_0.1.0.gz +# TODO: set to value below after migrating the script to Python 3 +# it now breaks because of urllib2 and SSL/TLS incompatibility when using Fastly +# RUST_DEPS_PACKAGES_URL = "https://rust-pkg-brave-core.s3.brave.com" RUST_DEPS_PACKAGES_URL = "https://s3-us-west-2.amazonaws.com/rust-pkg-brave-core" RUST_DEPS_PACKAGE_VERSION = "0.1.1"