From 82498226b411daa202d91b60fb0a924ab9fa5bc1 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 2 Mar 2021 16:03:00 -0700 Subject: [PATCH] [jenkins] convert baseline capture job to use tasks (#93288) Co-authored-by: spalger # Conflicts: # src/dev/ci_setup/setup.sh --- .ci/Jenkinsfile_baseline_capture | 62 +++++++++++++---------- src/dev/ci_setup/bootstrap_validations.sh | 45 ++++++++++++++++ src/dev/ci_setup/setup.sh | 46 ++--------------- 3 files changed, 84 insertions(+), 69 deletions(-) create mode 100644 src/dev/ci_setup/bootstrap_validations.sh diff --git a/.ci/Jenkinsfile_baseline_capture b/.ci/Jenkinsfile_baseline_capture index 6993dc9e087f97..672f5127a97964 100644 --- a/.ci/Jenkinsfile_baseline_capture +++ b/.ci/Jenkinsfile_baseline_capture @@ -3,40 +3,48 @@ library 'kibana-pipeline-library' kibanaLibrary.load() -kibanaPipeline(timeoutMinutes: 120) { +kibanaPipeline(timeoutMinutes: 210) { githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') { ciStats.trackBuild { - catchError { - withEnv([ - 'CI_PARALLEL_PROCESS_NUMBER=1' - ]) { - parallel([ - 'oss-baseline': { - workers.ci(name: 'oss-baseline', size: 'l', ramDisk: true, runErrorReporter: false, bootstrapped: false) { - // bootstrap ourselves, but with the env needed to upload the ts refs cache - withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { - withEnv([ - 'BUILD_TS_REFS_CACHE_ENABLE=true', - 'BUILD_TS_REFS_CACHE_CAPTURE=true' - ]) { - kibanaPipeline.doSetup() - } - } + catchErrors { + slackNotifications.onFailure( + title: "*<${env.BUILD_URL}|[${params.branch}] Baseline Capture Failure>*", + message: "[${params.branch}/${params.commit}] Baseline Capture Failure", + ) { + retryable.enable(2) - kibanaPipeline.functionalTestProcess('oss-baseline', './test/scripts/jenkins_baseline.sh')() + catchErrors { + workers.ci( + name: 'baseline-worker', + size: 'xl', + ramDisk: true, + runErrorReporter: false, + bootstrapped: false + ) { + withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { + withEnv([ + 'BUILD_TS_REFS_CACHE_ENABLE=true', + 'BUILD_TS_REFS_CACHE_CAPTURE=true', + 'DISABLE_BOOTSTRAP_VALIDATIONS=true', + ]) { + kibanaPipeline.doSetup() + } } - }, - 'xpack-baseline': { - workers.ci(name: 'xpack-baseline', size: 'l', ramDisk: true, runErrorReporter: false) { - kibanaPipeline.functionalTestProcess('xpack-baseline', './test/scripts/jenkins_xpack_baseline.sh')() + + kibanaPipeline.withCiTaskQueue([parallel: 2]) { + catchErrors { + tasks([ + kibanaPipeline.functionalTestProcess('oss-baseline', './test/scripts/jenkins_baseline.sh'), + kibanaPipeline.functionalTestProcess('xpack-baseline', './test/scripts/jenkins_xpack_baseline.sh'), + ]) + } } - }, - ]) + } + } } } - - kibanaPipeline.sendMail() - slackNotifications.onFailure() } + + kibanaPipeline.sendMail() } } diff --git a/src/dev/ci_setup/bootstrap_validations.sh b/src/dev/ci_setup/bootstrap_validations.sh new file mode 100644 index 00000000000000..5fc504aa5dffa2 --- /dev/null +++ b/src/dev/ci_setup/bootstrap_validations.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -e + +### +### verify no git modifications +### +GIT_CHANGES="$(git ls-files --modified)" +if [ "$GIT_CHANGES" ]; then + echo -e "\n${RED}ERROR: 'yarn kbn bootstrap' caused changes to the following files:${C_RESET}\n" + echo -e "$GIT_CHANGES\n" + exit 1 +fi + +### +### rebuild kbn-pm distributable to ensure it's not out of date +### +echo " -- building kbn-pm distributable" +yarn kbn run build -i @kbn/pm + +### +### verify no git modifications +### +GIT_CHANGES="$(git ls-files --modified)" +if [ "$GIT_CHANGES" ]; then + echo -e "\n${RED}ERROR: 'yarn kbn run build -i @kbn/pm' caused changes to the following files:${C_RESET}\n" + echo -e "$GIT_CHANGES\n" + exit 1 +fi + +### +### rebuild plugin list to ensure it's not out of date +### +echo " -- building plugin list docs" +node scripts/build_plugin_list_docs + +### +### verify no git modifications +### +GIT_CHANGES="$(git ls-files --modified)" +if [ "$GIT_CHANGES" ]; then + echo -e "\n${RED}ERROR: 'node scripts/build_plugin_list_docs' caused changes to the following files:${C_RESET}\n" + echo -e "$GIT_CHANGES\n" + exit 1 +fi diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index c4559029e5607a..c52d5a81517476 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -36,50 +36,12 @@ if [[ "$BUILD_TS_REFS_CACHE_CAPTURE" == "true" ]]; then cd "$KIBANA_DIR" fi +if [[ "$DISABLE_BOOTSTRAP_VALIDATIONS" != "true" ]]; then + source "$KIBANA_DIR/src/dev/ci_setup/bootstrap_validations.sh" +fi + ### ### Download es snapshots ### echo " -- downloading es snapshot" node scripts/es snapshot --download-only; - -### -### verify no git modifications -### -GIT_CHANGES="$(git ls-files --modified)" -if [ "$GIT_CHANGES" ]; then - echo -e "\n${RED}ERROR: 'yarn kbn bootstrap' caused changes to the following files:${C_RESET}\n" - echo -e "$GIT_CHANGES\n" - exit 1 -fi - -### -### rebuild kbn-pm distributable to ensure it's not out of date -### -echo " -- building kbn-pm distributable" -yarn kbn run build -i @kbn/pm - -### -### verify no git modifications -### -GIT_CHANGES="$(git ls-files --modified)" -if [ "$GIT_CHANGES" ]; then - echo -e "\n${RED}ERROR: 'yarn kbn run build -i @kbn/pm' caused changes to the following files:${C_RESET}\n" - echo -e "$GIT_CHANGES\n" - exit 1 -fi - -### -### rebuild plugin list to ensure it's not out of date -### -echo " -- building plugin list docs" -node scripts/build_plugin_list_docs - -### -### verify no git modifications -### -GIT_CHANGES="$(git ls-files --modified)" -if [ "$GIT_CHANGES" ]; then - echo -e "\n${RED}ERROR: 'node scripts/build_plugin_list_docs' caused changes to the following files:${C_RESET}\n" - echo -e "$GIT_CHANGES\n" - exit 1 -fi