diff --git a/Jenkinsfile b/Jenkinsfile index 439c6623e15..d43da8ea448 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,8 @@ pipeline { RUNBLD_DISABLE_NOTIFICATIONS = 'true' JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { timeout(time: 2, unit: 'HOURS') @@ -71,7 +73,7 @@ pipeline { stage('Lint'){ options { skipDefaultCheckout() } steps { - makeTarget("Lint", "check") + makeTarget(context: "Lint", target: "check") } } stage('Build and Test'){ @@ -91,7 +93,7 @@ pipeline { } } steps { - mageTarget("Elastic Agent x-pack Linux", "x-pack/elastic-agent", "build test") + mageTarget(context: "Elastic Agent x-pack Linux", directory: "x-pack/elastic-agent", target: "build test") } } @@ -105,7 +107,7 @@ pipeline { } } steps { - mageTargetWin("Elastic Agent x-pack Windows Unit test", "x-pack/elastic-agent", "build unitTest") + mageTargetWin(context: "Elastic Agent x-pack Windows Unit test", directory: "x-pack/elastic-agent", target: "build unitTest") } } @@ -119,7 +121,7 @@ pipeline { } } steps { - mageTarget("Elastic Agent x-pack Mac OS X", "x-pack/elastic-agent", "build unitTest") + mageTarget(context: "Elastic Agent x-pack Mac OS X", directory: "x-pack/elastic-agent", target: "build unitTest") } post { always { @@ -138,7 +140,7 @@ pipeline { } } steps { - makeTarget("Filebeat oss Linux", "-C filebeat testsuite") + makeTarget(context: "Filebeat oss Linux", target: "-C filebeat testsuite", withModule: true) } } stage('Filebeat x-pack'){ @@ -151,7 +153,7 @@ pipeline { } } steps { - mageTarget("Filebeat x-pack Linux", "x-pack/filebeat", "update build test") + mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "update build test", withModule: true) } } stage('Filebeat Mac OS X'){ @@ -164,7 +166,7 @@ pipeline { } } steps { - mageTarget("Filebeat oss Mac OS X", "filebeat", "build unitTest") + mageTarget(context: "Filebeat oss Mac OS X", directory: "filebeat", target: "build unitTest") } post { always { @@ -182,7 +184,7 @@ pipeline { } } steps { - mageTarget("Filebeat x-pack Mac OS X", "x-pack/filebeat", "build unitTest") + mageTarget(context: "Filebeat x-pack Mac OS X", directory: "x-pack/filebeat", target: "build unitTest") } post { always { @@ -200,7 +202,7 @@ pipeline { } } steps { - mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "build unitTest") + mageTargetWin(context: "Filebeat oss Windows Unit test", directory: "filebeat", target: "build unitTest") } } stage('Filebeat x-pack Windows'){ @@ -213,7 +215,7 @@ pipeline { } } steps { - mageTargetWin("Filebeat x-pack Windows", "x-pack/filebeat", "build unitTest") + mageTargetWin(context: "Filebeat x-pack Windows", directory: "x-pack/filebeat", target: "build unitTest") } } stage('Heartbeat'){ @@ -228,7 +230,7 @@ pipeline { stages { stage('Heartbeat oss'){ steps { - makeTarget("Heartbeat oss Linux", "-C heartbeat testsuite") + makeTarget(context: "Heartbeat oss Linux", target: "-C heartbeat testsuite") } } stage('Heartbeat Mac OS X'){ @@ -241,7 +243,7 @@ pipeline { } } steps { - mageTarget("Heartbeat oss Mac OS X", "heartbeat", "build unitTest") + mageTarget(context: "Heartbeat oss Mac OS X", directory: "heartbeat", target: "build unitTest") } post { always { @@ -259,7 +261,7 @@ pipeline { } } steps { - mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "build unitTest") + mageTargetWin(context: "Heartbeat oss Windows Unit test", directory: "heartbeat", target: "build unitTest") } } } @@ -274,7 +276,7 @@ pipeline { } } steps { - makeTarget("Auditbeat oss Linux", "-C auditbeat testsuite") + makeTarget(context: "Auditbeat oss Linux", target: "-C auditbeat testsuite", withModule: true) } } stage('Auditbeat crosscompile'){ @@ -287,7 +289,7 @@ pipeline { } } steps { - makeTarget("Auditbeat oss crosscompile", "-C auditbeat crosscompile") + makeTarget(context: "Auditbeat oss crosscompile", target: "-C auditbeat crosscompile") } } stage('Auditbeat oss Mac OS X'){ @@ -300,7 +302,7 @@ pipeline { } } steps { - mageTarget("Auditbeat oss Mac OS X", "auditbeat", "build unitTest") + mageTarget(context: "Auditbeat oss Mac OS X", directory: "auditbeat", target: "build unitTest") } post { always { @@ -318,7 +320,7 @@ pipeline { } } steps { - mageTargetWin("Auditbeat oss Windows Unit test", "auditbeat", "build unitTest") + mageTargetWin(context: "Auditbeat oss Windows Unit test", directory: "auditbeat", target: "build unitTest") } } stage('Auditbeat x-pack'){ @@ -331,7 +333,7 @@ pipeline { } } steps { - mageTarget("Auditbeat x-pack Linux", "x-pack/auditbeat", "update build test") + mageTarget(context: "Auditbeat x-pack Linux", directory: "x-pack/auditbeat", target: "update build test", withModule: true) } } stage('Auditbeat x-pack Mac OS X'){ @@ -344,7 +346,7 @@ pipeline { } } steps { - mageTarget("Auditbeat x-pack Mac OS X", "x-pack/auditbeat", "build unitTest") + mageTarget(context: "Auditbeat x-pack Mac OS X", directory: "x-pack/auditbeat", target: "build unitTest") } } stage('Auditbeat x-pack Windows'){ @@ -357,7 +359,7 @@ pipeline { } } steps { - mageTargetWin("Auditbeat x-pack Windows", "x-pack/auditbeat", "build unitTest") + mageTargetWin(context: "Auditbeat x-pack Windows", directory: "x-pack/auditbeat", target: "build unitTest") } } stage('Libbeat'){ @@ -372,17 +374,17 @@ pipeline { stages { stage('Libbeat oss'){ steps { - makeTarget("Libbeat oss Linux", "-C libbeat testsuite") + makeTarget(context: "Libbeat oss Linux", target: "-C libbeat testsuite") } } stage('Libbeat crosscompile'){ steps { - makeTarget("Libbeat oss crosscompile", "-C libbeat crosscompile") + makeTarget(context: "Libbeat oss crosscompile", target: "-C libbeat crosscompile") } } stage('Libbeat stress-tests'){ steps { - makeTarget("Libbeat stress-tests", "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") + makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") } } } @@ -397,7 +399,7 @@ pipeline { } } steps { - makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite") + makeTarget(context: "Libbeat x-pack Linux", target: "-C x-pack/libbeat testsuite") } } stage('Metricbeat OSS Unit tests'){ @@ -410,7 +412,7 @@ pipeline { } } steps { - mageTarget("Metricbeat OSS linux/amd64 (unitTest)", "metricbeat", "build unitTest") + mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") } } stage('Metricbeat OSS Integration tests'){ @@ -423,7 +425,7 @@ pipeline { } } steps { - mageTarget("Metricbeat OSS linux/amd64 (goIntegTest)", "metricbeat", "goIntegTest") + mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) } } stage('Metricbeat Python integration tests'){ @@ -436,7 +438,7 @@ pipeline { } } steps { - mageTarget("Metricbeat OSS linux/amd64 (pythonIntegTest)", "metricbeat", "pythonIntegTest") + mageTarget(context: "Metricbeat OSS linux/amd64 (pythonIntegTest)", directory: "metricbeat", target: "pythonIntegTest", withModule: true) } } stage('Metricbeat x-pack'){ @@ -463,7 +465,7 @@ pipeline { options { skipDefaultCheckout() } steps { withCloudTestEnv() { - mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test") + mageTarget(context: "Metricbeat x-pack Linux", directory: "x-pack/metricbeat", target: "build test", withModule: true) } } } @@ -484,7 +486,7 @@ pipeline { } } steps { - makeTarget("Metricbeat OSS crosscompile", "-C metricbeat crosscompile") + makeTarget(context: "Metricbeat OSS crosscompile", target: "-C metricbeat crosscompile") } } stage('Metricbeat Mac OS X'){ @@ -497,7 +499,7 @@ pipeline { } } steps { - mageTarget("Metricbeat OSS Mac OS X", "metricbeat", "build unitTest") + mageTarget(context: "Metricbeat OSS Mac OS X", directory: "metricbeat", target: "build unitTest") } } stage('Metricbeat x-pack Mac OS X'){ @@ -510,7 +512,7 @@ pipeline { } } steps { - mageTarget("Metricbeat x-pack Mac OS X", "x-pack/metricbeat", "build unitTest") + mageTarget(context: "Metricbeat x-pack Mac OS X", directory: "x-pack/metricbeat", target: "build unitTest") } post { always { @@ -528,7 +530,7 @@ pipeline { } } steps { - mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "build unitTest") + mageTargetWin(context: "Metricbeat Windows Unit test", directory: "metricbeat", target: "build unitTest") } } stage('Metricbeat x-pack Windows'){ @@ -541,7 +543,7 @@ pipeline { } } steps { - mageTargetWin("Metricbeat x-pack Windows", "x-pack/metricbeat", "build unitTest") + mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") } } stage('Packetbeat'){ @@ -556,7 +558,7 @@ pipeline { stages { stage('Packetbeat oss'){ steps { - makeTarget("Packetbeat oss Linux", "-C packetbeat testsuite") + makeTarget(context: "Packetbeat oss Linux", target: "-C packetbeat testsuite") } } } @@ -573,7 +575,7 @@ pipeline { stages { stage('Dockerlogbeat'){ steps { - mageTarget("Elastic Docker Logging Driver Plugin unit tests", "x-pack/dockerlogbeat", "update build test") + mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "update build test") } } } @@ -590,7 +592,7 @@ pipeline { stages { stage('Winlogbeat oss'){ steps { - makeTarget("Winlogbeat oss crosscompile", "-C winlogbeat crosscompile") + makeTarget(context: "Winlogbeat oss crosscompile", target: "-C winlogbeat crosscompile") } } stage('Winlogbeat Windows'){ @@ -603,7 +605,7 @@ pipeline { } } steps { - mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "build unitTest") + mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "winlogbeat", target: "build unitTest") } } } @@ -618,7 +620,7 @@ pipeline { } } steps { - mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "build unitTest") + mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "x-pack/winlogbeat", target: "build unitTest", withModule: true) } } stage('Functionbeat'){ @@ -633,9 +635,9 @@ pipeline { stages { stage('Functionbeat x-pack'){ steps { - mageTarget("Functionbeat x-pack Linux", "x-pack/functionbeat", "update build test") + mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") withEnv(["GO_VERSION=1.13.1"]){ - makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat test-gcp-functions") + makeTarget(context: "Functionbeat x-pack Linux", target: "-C x-pack/functionbeat test-gcp-functions") } } } @@ -649,7 +651,7 @@ pipeline { } } steps { - mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "build unitTest") + mageTarget(context: "Functionbeat x-pack Mac OS X", directory: "x-pack/functionbeat", target: "build unitTest") } post { always { @@ -667,7 +669,7 @@ pipeline { } } steps { - mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "build unitTest") + mageTargetWin(context: "Functionbeat Windows Unit test", directory: "x-pack/functionbeat", target: "build unitTest") } } } @@ -684,7 +686,7 @@ pipeline { stages { stage('Journalbeat oss'){ steps { - makeTarget("Journalbeat Linux", "-C journalbeat testsuite") + makeTarget(context: "Journalbeat Linux", target: "-C journalbeat testsuite") } } } @@ -703,8 +705,8 @@ pipeline { steps { // FIXME see https://github.com/elastic/beats/issues/18132 catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { - makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") - makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") + makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test") + makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test-package") } } } @@ -712,8 +714,8 @@ pipeline { steps { // FIXME see https://github.com/elastic/beats/issues/18132 catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { - makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") - makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") + makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test") + makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test-package") } } } @@ -729,7 +731,7 @@ pipeline { steps { // FIXME see https://github.com/elastic/beats/issues/18132 catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { - makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") + makeTarget(context: "Generators Metricbeat Mac OS X", target: "-C generator/_templates/metricbeat test") } } post { @@ -750,7 +752,7 @@ pipeline { steps { // FIXME see https://github.com/elastic/beats/issues/18132 catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { - makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") + makeTarget(context: "Generators Beat Mac OS X", target: "-C generator/_templates/beat test") } } post { @@ -801,9 +803,13 @@ def fixPermissions(location) { script/fix_permissions.sh ${location}""", returnStatus: true) } -def makeTarget(String context, String target, boolean clean = true) { +def makeTarget(Map args = [:]) { + def context = args.context + def target = args.target + def clean = args.get('clean', true) + def withModule = args.get('withModule', false) withGithubNotify(context: "${context}") { - withBeatsEnv(true) { + withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(target)) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMage() @@ -816,9 +822,13 @@ def makeTarget(String context, String target, boolean clean = true) { } } -def mageTarget(String context, String directory, String target) { +def mageTarget(Map args = [:]) { + def context = args.context + def directory = args.directory + def target = args.target + def withModule = args.get('withModule', false) withGithubNotify(context: "${context}") { - withBeatsEnv(true) { + withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(directory)) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMage() @@ -832,9 +842,13 @@ def mageTarget(String context, String directory, String target) { } } -def mageTargetWin(String context, String directory, String target) { +def mageTargetWin(Map args = [:]) { + def context = args.context + def directory = args.directory + def target = args.target + def withModule = args.get('withModule', false) withGithubNotify(context: "${context}") { - withBeatsEnvWin() { + withBeatsEnvWin(withModule: withModule, modulePattern: getModulePattern(directory)) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMageWin() @@ -848,10 +862,27 @@ def mageTargetWin(String context, String directory, String target) { } } -def withBeatsEnv(boolean archive, Closure body) { +def getModulePattern(String toCompare) { + // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') + return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) +} + +def withBeatsEnv(Map args = [:], Closure body) { + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def modulePattern + if (withModule) { + modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnv: modulePattern parameter is required.') + } def os = goos() def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : '' + withEnv([ "HOME=${env.WORKSPACE}", "GOPATH=${env.WORKSPACE}", @@ -863,9 +894,8 @@ def withBeatsEnv(boolean archive, Closure body) { "PYTHON_ENV=${WORKSPACE}/python-env", "TEST_TAGS=${env.TEST_TAGS},oracle", "DOCKER_PULL=0", + "MODULE=${module}" ]) { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") if(isDockerInstalled()){ dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } @@ -891,11 +921,22 @@ def withBeatsEnv(boolean archive, Closure body) { } } -def withBeatsEnvWin(Closure body) { +def withBeatsEnvWin(Map args = [:], Closure body) { + def withModule = args.get('withModule', false) + def modulePattern + if (withModule) { + modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnvWin: modulePattern parameter is required.') + } final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : '' + withEnv([ "HOME=${env.WORKSPACE}", "GOPATH=${env.WORKSPACE}", @@ -904,9 +945,8 @@ def withBeatsEnvWin(Closure body) { "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", "TEST_COVERAGE=true", "RACE_DETECTOR=true", + "MODULE=${module}" ]){ - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${env.BASE_DIR}"){ installTools() try { @@ -1020,7 +1060,7 @@ def k8sTest(versions){ stage("k8s ${v}"){ withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(false) { + withBeatsEnv(archive: false, withModule: false) { sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") @@ -1144,7 +1184,7 @@ def terraformApply(String directory) { // terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') def startCloudTestEnv(String name, environments = []) { withCloudTestEnv() { - withBeatsEnv(false) { + withBeatsEnv(archive: false, withModule: false) { def runAll = params.runAllCloudTests try { for (environment in environments) { @@ -1169,7 +1209,7 @@ def startCloudTestEnv(String name, environments = []) { def terraformCleanup(String stashName, String directory) { stage("Remove cloud scenarios in ${directory}"){ withCloudTestEnv() { - withBeatsEnv(false) { + withBeatsEnv(archive: false, withModule: false) { unstash("terraform-${stashName}") retry(2) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") @@ -1252,11 +1292,21 @@ def loadConfigEnvVars(){ // Skip all the stages for changes only related to the documentation env.ONLY_DOCS = isDocChangedOnly() +} + +/** + This method gathers the module name, if required, in order to run the ITs only if + the changeset affects a specific module. - // Run the ITs by running only if the changeset affects a specific module. - // For such, it's required to look for changes under the module folder and exclude anything else - // such as ascidoc and png files. - env.MODULE = getGitMatchingGroup(pattern: '[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*', exclude: '^(((?!\\/module\\/).)*$|.*\\.asciidoc|.*\\.png)') + For such, it's required to look for changes under the module folder and exclude anything else + such as ascidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String pattern) { + def module = '' + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern , exclude: '^(((?!\\/module\\/).)*$|.*\\.asciidoc|.*\\.png)') + } + return module } /**