diff --git a/.ci/jobs/beats-mbp-2.0.yml b/.ci/jobs/beats-mbp-2.0.yml deleted file mode 100644 index 3ccc435c8bd..00000000000 --- a/.ci/jobs/beats-mbp-2.0.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -- job: - name: Beats/beats-mbp-2.0 - display-name: 'Beats (2.0)' - description: 'Beats Main Pipeline 2.0' - view: Beats - concurrent: true - project-type: multibranch - prune-dead-branches: true - days-to-keep: 30 - script-path: '.ci/Jenkinsfile' - triggers: [] - wrappers: [] - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' - discover-tags: true - disable-pr-notifications: true - notification-context: "beats-ci-2.0" - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: 365 - - change-request: - ignore-target-only-changes: true - - named-branches: - - exact-name: - name: 'master' - case-sensitive: true - - regex-name: - regex: '7\.[x789]' - case-sensitive: true - - regex-name: - regex: '8\.\d+' - case-sensitive: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 3 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 145d7ae09e5..01517e07245 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630] - Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685] - Change network.direction values to ECS recommended values (inbound, outbound). {issue}12445[12445] {pull}20695[20695] +- Docker container needs to be explicitly run as user root for auditing. {pull}21202[21202] *Filebeat* diff --git a/Jenkinsfile b/Jenkinsfile index b0f90c07b0b..e7f91ef95e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,20 +3,6 @@ @Library('apm@current') _ import groovy.transform.Field - - -/** - NOTE: Important note regarding the agents and labels. - agent labels are defined in the gobld service, that's managed by infra. The required format - is: - - ' && immutable' for linux OS. - - 'macosx' for the MacOS. - - 'windows-immutable && windows-' for Windows. NOTE: version might differ in some cases - - The above labels will help to set what OS family and specific version of the agent is - required to used in the stage. -*/ - /** This is required to store the stashed id with the test results to be digested with runbld */ @@ -25,20 +11,19 @@ import groovy.transform.Field pipeline { agent { label 'ubuntu-18 && immutable' } environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' BASE_DIR = 'src/github.com/elastic/beats' - GOX_FLAGS = "-arch amd64" - DOCKER_COMPOSE_VERSION = "1.21.0" - TERRAFORM_VERSION = "0.12.24" - PIPELINE_LOG_LEVEL = "INFO" DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' + GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}" + PIPELINE_LOG_LEVEL = 'INFO' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { timeout(time: 2, unit: 'HOURS') @@ -51,23 +36,16 @@ pipeline { rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) } triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test(\\W+macos)?$)') + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') } parameters { - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') - booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') - string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') } stages { - /** - Checkout the code and stash it, to use it on other stages. - */ stage('Checkout') { options { skipDefaultCheckout() } steps { @@ -76,711 +54,65 @@ pipeline { gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ - loadConfigEnvVars() - } - whenTrue(params.debug){ - dumpFilteredEnvironment() + // Skip all the stages except docs for PR's with asciidoc and md changes only + setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) + setEnvVar('GO_VERSION', readFile(".go-version").trim()) + withEnv(["HOME=${env.WORKSPACE}"]) { + retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } + } } } } stage('Lint'){ options { skipDefaultCheckout() } environment { - // See https://github.com/elastic/beats/pull/19823 GOFLAGS = '-mod=readonly' } steps { - makeTarget(context: "Lint", target: "check") + withGithubNotify(context: 'Lint') { + withBeatsEnv(archive: true) { + dumpVariables() + cmd(label: 'make check', script: 'make check') + } + } } } - stage('Build and Test'){ + stage('Build&Test') { + options { skipDefaultCheckout() } when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - parallel { - stage('Elastic Agent x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Linux", directory: "x-pack/elastic-agent", target: "build test") - } - } - stage('Elastic Agent x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Elastic Agent x-pack Windows Unit test", directory: "x-pack/elastic-agent", target: "build unitTest") - } - } - stage('Elastic Agent Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Mac OS X", directory: "x-pack/elastic-agent", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" - } - } - steps { - mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat oss Mac OS X", directory: "filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat x-pack Mac OS X", directory: "x-pack/filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat oss Windows Unit test", directory: "filebeat", target: "build unitTest") - } - } - stage('Filebeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat x-pack Windows", directory: "x-pack/filebeat", target: "build unitTest") - } - } - stage('Heartbeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") - } - } - stage('Heartbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTarget(context: "Heartbeat oss Mac OS X", directory: "heartbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Heartbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTargetWin(context: "Heartbeat oss Windows Unit test", directory: "heartbeat", target: "build unitTest") - } - } - stage('Auditbeat oss Linux'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") - } - } - stage('Auditbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile") - } - } - stage('Auditbeat oss Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat oss Mac OS X", directory: "auditbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Auditbeat oss Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat oss Windows Unit test", directory: "auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Auditbeat x-pack Linux", directory: "x-pack/auditbeat", target: "update build test", withModule: true) - } - } - stage('Auditbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat x-pack Mac OS X", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat x-pack Windows", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Libbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT != "false" - } - } - stages { - stage('Libbeat oss'){ - steps { - mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") - } - } - stage('Libbeat crosscompile'){ - steps { - makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile") - } - } - stage('Libbeat stress-tests'){ - steps { - makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") - } - } - } - } - stage('Libbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() } + expression { return params.runAllStages } // If UI forced } - stage('Metricbeat OSS Unit tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat OSS Go Integration tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) - } - } - stage('Metricbeat OSS Python Integration tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (pythonIntegTest)", directory: "metricbeat", target: "pythonIntegTest", withModule: true) - } - } - stage('Metricbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" - } - } - stages { - stage('Prepare cloud integration tests environments'){ - options { skipDefaultCheckout() } - steps { - startCloudTestEnv('x-pack-metricbeat', [ - [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], - ]) - } - } - stage('Metricbeat x-pack'){ - options { skipDefaultCheckout() } - steps { - withCloudTestEnv() { - mageTarget(context: "Metricbeat x-pack Linux", directory: "x-pack/metricbeat", target: "build test", withModule: true) - } - } - } - } - post { - cleanup { - terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') - } - } - } - stage('Metricbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile") - } - } - stage('Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat OSS Mac OS X", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat x-pack Mac OS X", directory: "x-pack/metricbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Metricbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat Windows Unit test", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") - } - } - stage('Packetbeat Linux'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") - } - } - stage('Packetbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Packetbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") - } - } - stage('dockerlogbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_DOCKERLOGBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") - } - } - stage('Winlogbeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_WINLOGBEAT != "false" - } - } - steps { - makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile") - } - } - stage('Winlogbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_WINLOGBEAT != "false" - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "winlogbeat", target: "build unitTest") - } - } - stage('Winlogbeat Windows x-pack'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "x-pack/winlogbeat", target: "build unitTest", withModule: true) - } - } - stage('Functionbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") - withEnv(["GO_VERSION=1.13.1"]){ - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") - } - } - } - stage('Functionbeat Mac OS X x-pack'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Functionbeat x-pack Mac OS X", directory: "x-pack/functionbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Functionbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTargetWin(context: "Functionbeat Windows Unit test", directory: "x-pack/functionbeat", target: "build unitTest") - } - } - stage('Journalbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_JOURNALBEAT != "false" - } - } - steps { - mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build unitTest") - } - } - stage('Generators'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_GENERATOR != "false" - } - } - stages { - stage('Generators Metricbeat Linux'){ - steps { - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test") - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package") - } - } - stage('Generators Beat Linux'){ - steps { - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test") - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package") + } + steps { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + script { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> + mapParallelTasks["${k}"] = v } } + parallel(mapParallelTasks) } } - stage('Generators Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_GENERATOR != "false" - } - } - steps { - makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test") - } - post { - always { - delete() - } - } - } - stage('Generators Beat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_GENERATOR != "false" - } - } - steps { - makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test") - } - post { - always { - delete() - } - } - } - stage('Kubernetes'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_KUBERNETES != "false" - } - } - steps { - k8sTest(["v1.18.2","v1.17.2","v1.16.4","v1.15.7","v1.14.10"]) + } + post { + always { + dir("${BASE_DIR}"){ + // Archive the markdown files that contain the build reasons + archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md') } } } @@ -796,131 +128,158 @@ pipeline { } } -def delete() { - dir("${env.BASE_DIR}") { - fixPermissions("${WORKSPACE}") +/** +* This method is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +*/ +def generateStages(Map args = [:]) { + def projectName = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def fileName = "${projectName}/Jenkinsfile.yml" + if (fileExists(fileName)) { + def content = readYaml(file: fileName) + // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) + } + } else { + log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") } - deleteDir() -} - -def fixPermissions(location) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) + return mapParallelStages } -def makeTarget(Map args = [:]) { - def context = args.context - def target = args.target - def directory = args.get('directory', '') - def clean = args.get('clean', true) - def withModule = args.get('withModule', false) - def directoryFlag = directory.trim() ? "-C ${directory}" : '' - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") - whenTrue(clean) { - fixPermissions("${HOME}") - } +def cloud(Map args = [:]) { + node(args.label) { + startCloudTestEnv(name: args.directory, dirs: args.dirs) + } + withCloudTestEnv() { + try { + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) + } finally { + terraformCleanup(name: args.directory, dir: args.directory) } } } -def mageTarget(Map args = [:]) { - def context = args.context - def directory = args.directory - def target = args.target - def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") +def k8sTest(Map args = [:]) { + def versions = args.versions + node(args.label) { + versions.each{ v -> + stage("${args.context} ${v}"){ + withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ + withGithubNotify(context: "${args.context} ${v}") { + withBeatsEnv(archive: false, withModule: false) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } + try { + sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") + sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") + sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") + } finally { + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } + } } } } } -def mageTargetWin(Map args = [:]) { +/** +* This method runs the given command supporting two kind of scenarios: +* - make -C then the dir(location) is not required, aka by disaling isMage: false +* - mage then the dir(location) is required, aka by enabling isMage: true. +*/ +def target(Map args = [:]) { def context = args.context - def directory = args.directory - def target = args.target + def command = args.command + def directory = args.get('directory', '') def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnvWin(withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMageWin() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + def isMage = args.get('isMage', false) + node(args.label) { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + dumpVariables() + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + cmd(label: "${command}", script: "${command}") + } } } } } -def getModulePattern(String toCompare) { - // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') - return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) -} - +/** +* This method wraps all the environment setup and pre-requirements to run any commands. +*/ def withBeatsEnv(Map args = [:], Closure body) { def archive = args.get('archive', true) def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) + + def goRoot, path, magefile, pythonEnv, testResults, artifacts + + if(isUnix()) { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" + magefile = "${WORKSPACE}/.magefile" + pythonEnv = "${WORKSPACE}/python-env" + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' + } else { + def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}" + magefile = "${env.WORKSPACE}\\.magefile" + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" } - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' withEnv([ - "HOME=${env.WORKSPACE}", + "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - "MAGEFILE_CACHE=${WORKSPACE}/.magefile", - "TEST_COVERAGE=true", + "HOME=${env.WORKSPACE}", + "MAGEFILE_CACHE=${magefile}", + "MODULE=${module}", + "PATH=${path}", + "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", - "PYTHON_ENV=${WORKSPACE}/python-env", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "DOCKER_PULL=0", - "MODULE=${module}" + "TEST_COVERAGE=true", + "TEST_TAGS=${env.TEST_TAGS},oracle" ]) { - if(isDockerInstalled()){ + if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { installTools() - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - setGitConfig() + if(isUnix()) { + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config user.email "beatsmachine@users.noreply.github.com" + git config user.name "beatsmachine" + fi''') + } try { - if(!params.dry_run){ - body() - } + body() } finally { if (archive) { - archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out') + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) + } + // Tear down the setup for the permamnent workers. + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + fixPermissions("${WORKSPACE}") + deleteDir() } } } @@ -928,8 +287,57 @@ def withBeatsEnv(Map args = [:], Closure body) { } /** - This method archives and report the tests output, for such, it searches in certain folders - to bypass some issues when working with big repositories. +* This method fixes the filesystem permissions after the build has happenend. The reason is to +* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental +* issues. +*/ +def fixPermissions(location) { + if(isUnix()) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + set +x + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${location}""", returnStatus: true) + } +} + +/** +* This method installs the required dependencies that are for some reason not available in the +* CI Workers. +*/ +def installTools() { + if(isUnix()) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } + } else { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } +} + +/** +* This method gathers the module name, if required, in order to run the ITs only if +* the changeset affects a specific module. +* +* For such, it's required to look for changes under the module folder and exclude anything else +* such as asciidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String directory) { + // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') + def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) + def module = '' + + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) + } + return module +} + +/** +* This method archives and report the tests output, for such, it searches in certain folders +* to bypass some issues when working with big repositories. */ def archiveTestOutput(Map args = [:]) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { @@ -938,222 +346,68 @@ def archiveTestOutput(Map args = [:]) { } cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() tar(file: "${name}.tgz", archive: true, dir: folder) } } } } -def withBeatsEnvWin(Map args = [:], Closure body) { - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) - } - final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" - - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - - withEnv([ - "HOME=${env.WORKSPACE}", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", - "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - "MODULE=${module}" - ]){ - dir("${env.BASE_DIR}"){ - installTools() - try { - if(!params.dry_run){ - body() - } - } finally { - archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out") - } - } - } -} - -def installTools() { - def i = 2 // Number of retries - if(isUnix()) { - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") } - } else { - // Install python3 with the specific step, even though install-tools.bat will verify if it's there anyway. - // TODO: as soon as python3 is installed in the CI Workers we will be able to remove the line below. - retryWithSleep(retries: i, seconds: 5, backoff: true){ installTools([ [tool: 'python3', version: '3.8', exclude: 'rc'] ]) } - retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } - } -} - -def goos(){ - def labels = env.NODE_LABELS - - if (labels.contains('linux')) { - return 'linux' - } else if (labels.contains('windows')) { - return 'windows' - } else if (labels.contains('darwin')) { - return 'darwin' - } - - error("Unhandled OS name in NODE_LABELS: " + labels) -} - -def dumpMage(){ - echo "### MAGE DUMP ###" - sh(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpMageWin(){ - echo "### MAGE DUMP ###" - bat(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpFilteredEnvironment(){ - echo "### ENV DUMP ###" - echo "PATH: ${env.PATH}" - echo "HOME: ${env.HOME}" - echo "USERPROFILE: ${env.USERPROFILE}" - echo "BUILD_DIR: ${env.BUILD_DIR}" - echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" - echo "BEATS: ${env.BEATS}" - echo "PROJECTS: ${env.PROJECTS}" - echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" - echo "PYTHON_ENV: ${env.PYTHON_ENV}" - echo "PYTHON_EXE: ${env.PYTHON_EXE}" - echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" - echo "VENV_PARAMS: ${env.VENV_PARAMS}" - echo "FIND: ${env.FIND}" - echo "GOLINT: ${env.GOLINT}" - echo "GOLINT_REPO: ${env.GOLINT_REPO}" - echo "REVIEWDOG: ${env.REVIEWDOG}" - echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" - echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" - echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" - echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" - echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" - echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" - echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" - echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" - echo "NOW: ${env.NOW}" - echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" - echo "GOIMPORTS: ${env.GOIMPORTS}" - echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" - echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" - echo "PROCESSES: ${env.PROCESSES}" - echo "TIMEOUT: ${env.TIMEOUT}" - echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" - echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" - echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" - echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" - echo "STRESS_TESTS: ${env.STRESS_TESTS}" - echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" - echo "TEST_TAGS: ${env.TEST_TAGS}" - echo "GOX_OS: ${env.GOX_OS}" - echo "GOX_OSARCH: ${env.GOX_OSARCH}" - echo "GOX_FLAGS: ${env.GOX_FLAGS}" - echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" - echo "BEAT_VERSION: ${env.BEAT_VERSION}" - echo "COMMIT_ID: ${env.COMMIT_ID}" - echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" - echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" - echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" - echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" - echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" - echo "### END ENV DUMP ###" +/** +* This method wraps the junit built-in step to archive the test reports that gonna be populated later on +* with the runbld post build step. +*/ +def junitAndStore(Map args = [:]) { + junit(args) + // args.id could be null in some cases, so let's use the currentmilliseconds + def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" + stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) + stashedTestReports[stageName] = stageName } -def k8sTest(versions){ - versions.each{ v -> - stage("k8s ${v}"){ - withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(archive: false, withModule: false) { - sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") - sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - sh(label: 'Delete cluster', script: 'kind delete cluster') +/** +* This method populates the test output using the runbld approach. For such it requires the +* global variable stashedTestReports. +* TODO: should be moved to the shared library +*/ +def runbld() { + catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { + if (stashedTestReports) { + def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${env.BASE_DIR}") { + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) } } } + sh(label: 'Process JUnit reports with runbld', + script: """\ + ## for debugging purposes + find . -name "TEST-*.xml" + cat >./runbld-script </.*`. -* -* In addition, there are another two alternatives to report that there are -* changes, when `runAllStages` parameter is set to true or when running on a -* branch/tag basis. +* This method executes a closure with credentials for cloud test +* environments. */ -def isChanged(patterns){ - return ( - params.runAllStages // when runAllStages UI parameter is set to true - || !isPR() // when running on a branch/tag - || isGitRegionMatch(patterns: patterns, comparator: 'regexp') - ) -} - -def isChangedOSSCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^pytest.ini", - "^libbeat/.*", - "^testing/.*", - "^dev-tools/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -def isChangedXPackCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^pytest.ini", - "^libbeat/.*", - "^dev-tools/.*", - "^testing/.*", - "^x-pack/libbeat/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -// withCloudTestEnv executes a closure with credentials for cloud test -// environments. def withCloudTestEnv(Closure body) { def maskedVars = [] def testTags = "${env.TEST_TAGS}" @@ -1184,58 +438,60 @@ def withCloudTestEnv(Closure body) { } } -def terraformInit(String directory) { - dir(directory) { - sh(label: "Terraform Init on ${directory}", script: "terraform init") - } -} - -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } -} - -// Start testing environment on cloud using terraform. Terraform files are -// stashed so they can be used by other stages. They are also archived in -// case manual cleanup is needed. -// -// Example: -// startCloudTestEnv('x-pack-metricbeat', [ -// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], -// ]) -// ... -// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') -def startCloudTestEnv(String name, environments = []) { - withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { - def runAll = params.runAllCloudTests - try { - for (environment in environments) { - if (environment.cond || runAll) { +/** +* Start testing environment on cloud using terraform. Terraform files are +* stashed so they can be used by other stages. They are also archived in +* case manual cleanup is needed. +* +* Example: +* startCloudTestEnv(name: 'x-pack-metricbeat', dirs: ['x-pack/metricbeat/module/aws']) +* ... +* terraformCleanup(name: 'x-pack-metricbeat', dir: 'x-pack/metricbeat') +*/ +def startCloudTestEnv(Map args = [:]) { + String name = normalise(args.name) + def dirs = args.get('dirs',[]) + stage("${name}-prepare-cloud-env"){ + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + try { + for (folder in dirs) { retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(environment.dir) + terraformApply(folder) } } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } } } +/** +* Run terraform in the given directory +*/ +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} -// Looks for all terraform states in directory and runs terraform destroy for them, -// it uses terraform states previously stashed by startCloudTestEnv. -def terraformCleanup(String stashName, String directory) { - stage("Remove cloud scenarios in ${directory}"){ +/** +* Tear down the terraform environments, by looking for all terraform states in directory +* then it runs terraform destroy for each one. +* It uses terraform states previously stashed by startCloudTestEnv. +*/ +def terraformCleanup(Map args = [:]) { + String name = normalise(args.name) + String directory = args.dir + stage("${name}-tear-down-cloud-env"){ withCloudTestEnv() { withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${stashName}") + unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") } @@ -1244,184 +500,138 @@ def terraformCleanup(String stashName, String directory) { } } -def loadConfigEnvVars(){ - def empty = [] - env.GO_VERSION = readFile(".go-version").trim() - - withEnv(["HOME=${env.WORKSPACE}"]) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") } - } - - // Libbeat is the core framework of Beats. It has no additional dependencies - // on other projects in the Beats repository. - env.BUILD_LIBBEAT = isChangedOSSCode(empty) - env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) - - // Auditbeat depends on metricbeat as framework, but does not include any of - // the modules from Metricbeat. - // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) - - // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) - - // Filebeat depends on libbeat only. - // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) - - // Metricbeat depends on libbeat only. - // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) - - // Functionbeat is a standalone beat that depends on libbeat only. - // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) - - // Heartbeat depends on libbeat only. - // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) - - // Journalbeat depends on libbeat only. - // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) - - // Packetbeat depends on libbeat only. - // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) - - // Winlogbeat depends on libbeat only. - // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) - - // Elastic-agent is a self-contained product, that depends on libbeat only. - // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. - // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) - - // The Kubernetes test use Filebeat and Metricbeat, but only need to be run - // if the deployment scripts have been updated. No Beats specific testing is - // involved. - env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - - def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) - generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) - env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) - - // Skip all the stages for changes only related to the documentation - env.ONLY_DOCS = isDocChangedOnly() - - // Enable macOS builds when required - env.BUILD_ON_MACOS = (params.macosTest // UI Input parameter is set to true - || !isPR() // For branches and tags - || matchesPrLabel(label: 'macOS') // If `macOS` GH label (Case-Sensitive) - || (env.GITHUB_COMMENT?.toLowerCase()?.contains('/test macos'))) // If `/test macos` in the GH comment (Case-Insensitive) -} - /** - This method gathers the module name, if required, in order to run the ITs only if - the changeset affects a specific module. - - For such, it's required to look for changes under the module folder and exclude anything else - such as ascidoc and png files. +* Prepare the terraform context in the given directory */ -def getCommonModuleInTheChangeSet(String pattern, String directory) { - def module = '' - // Transform folder structure in regex format since path separator is required to be escaped - def transformedDirectory = directory.replaceAll('/', '\\/') - def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" - def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" - dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern, exclude: exclude) +def terraformInit(String directory) { + dir(directory) { + sh(label: "Terraform Init on ${directory}", script: "terraform init") } - return module } /** - This method verifies if the changeset for the current pull request affect only changes related - to documentation, such as asciidoc and png files. +* Replace the slashes in the directory in case there are nested folders. */ -def isDocChangedOnly(){ - if (params.runAllStages || !env.CHANGE_ID?.trim()) { - log(level: 'INFO', text: 'Speed build for docs only is disabled for branches/tags or when forcing with the runAllStages parameter.') - return 'false' - } else { - log(level: "INFO", text: 'Check if the speed build for docs is enabled.') - return isGitRegionMatch(patterns: ['.*\\.(asciidoc|png)'], shouldMatchAll: true) - } +def normalise(String directory) { + return directory.replaceAll("[\\W]|_",'-') } /** - This method grab the dependencies of a Go module and transform them on regexp +* For debugging purposes. */ -def getProjectDependencies(beatName){ - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - def output = "" - - withEnv([ - "HOME=${env.WORKSPACE}/${env.BASE_DIR}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - ]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) - } - return output?.split('\n').collect{ item -> item as String } -} - -def setGitConfig(){ - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" - fi - ''') +def dumpVariables(){ + echo "### MAGE DUMP ###" + cmd(label: 'Dump mage variables', script: 'mage dumpVariables') + echo "### END MAGE DUMP ###" + echo """ + ### ENV DUMP ### + BEAT_VERSION: ${env.BEAT_VERSION} + BEATS: ${env.BEATS} + BUILD_DIR: ${env.BUILD_DIR} + COMMIT_ID: ${env.COMMIT_ID} + COVERAGE_DIR: ${env.COVERAGE_DIR} + COVERAGE_TOOL: ${env.COVERAGE_TOOL} + COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO} + DOCKER_CACHE: ${env.DOCKER_CACHE} + DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME} + DOCKER_COMPOSE: ${env.DOCKER_COMPOSE} + FIND: ${env.FIND} + GOBUILD_FLAGS: ${env.GOBUILD_FLAGS} + GOIMPORTS: ${env.GOIMPORTS} + GOIMPORTS_REPO: ${env.GOIMPORTS_REPO} + GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX} + GOLINT: ${env.GOLINT} + GOLINT_REPO: ${env.GOLINT_REPO} + GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP} + GOX_FLAGS: ${env.GOX_FLAGS} + GOX_OS: ${env.GOX_OS} + GOX_OSARCH: ${env.GOX_OSARCH} + HOME: ${env.HOME} + NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS} + NOW: ${env.NOW} + PATH: ${env.PATH} + PKG_BUILD_DIR: ${env.PKG_BUILD_DIR} + PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR} + PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS} + PROJECTS: ${env.PROJECTS} + PROJECTS_ENV: ${env.PROJECTS_ENV} + PYTHON_ENV: ${env.PYTHON_ENV} + PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE} + PYTHON_EXE: ${env.PYTHON_EXE} + PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES} + PROCESSES: ${env.PROCESSES} + REVIEWDOG: ${env.REVIEWDOG} + REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS} + REVIEWDOG_REPO: ${env.REVIEWDOG_REPO} + STRESS_TESTS: ${env.STRESS_TESTS} + STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS} + SYSTEM_TESTS: ${env.SYSTEM_TESTS} + TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO} + TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT} + TEST_TAGS: ${env.TEST_TAGS} + TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT} + TIMEOUT: ${env.TIMEOUT} + USERPROFILE: ${env.USERPROFILE} + VENV_PARAMS: ${env.VENV_PARAMS} + XPACK_SUFFIX: ${env.XPACK_SUFFIX} + ### END ENV DUMP ### + """ } def isDockerInstalled(){ - return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + if (isUnix()) { + // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) { + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + } else { + return false + } } -def junitAndStore(Map params = [:]){ - junit(params) - // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds - def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" - stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) - stashedTestReports[stageName] = stageName +/** +* This class is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +* +* What parameters/arguments are supported: +* - label -> the worker labels +* - project -> the name of the project that should match with the folder name. +* - content -> the specific stage data in the /Jenkinsfile.yml +* - context -> the name of the stage, normally -(-)? +*/ +class RunCommand extends co.elastic.beats.BeatsFunction { + public RunCommand(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def withModule = args.content.get('withModule', false) + if(args?.content?.containsKey('make')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('mage')) { + steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) + } + if(args?.content?.containsKey('k8sTest')) { + steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) + } + if(args?.content?.containsKey('cloud')) { + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) + } + } } -def runbld() { - catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { - if (stashedTestReports) { - def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${env.BASE_DIR}") { - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } - } - } - sh(label: 'Process JUnit reports with runbld', - script: """\ - cat >./runbld-script < item as String } } } diff --git a/auditbeat/docs/running-on-docker.asciidoc b/auditbeat/docs/running-on-docker.asciidoc index 74007cdeb35..dee50fa254a 100644 --- a/auditbeat/docs/running-on-docker.asciidoc +++ b/auditbeat/docs/running-on-docker.asciidoc @@ -10,5 +10,5 @@ It is also essential to run {beatname_uc} in the host PID namespace. ["source","sh",subs="attributes"] ---- -docker run --cap-add=AUDIT_CONTROL,AUDIT_READ --pid=host {dockerimage} +docker run --cap-add=AUDIT_CONTROL --cap-add=AUDIT_READ --user=root --pid=host {dockerimage} ---- diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 73110b17354..bc99856a890 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -92,7 +92,7 @@ func Package() { // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages(devtools.WithRootUserContainer()) + return devtools.TestPackages() } // Update is an alias for running fields, dashboards, config, includes. diff --git a/auditbeat/scripts/mage/package.go b/auditbeat/scripts/mage/package.go index fbda2077f4f..09591705121 100644 --- a/auditbeat/scripts/mage/package.go +++ b/auditbeat/scripts/mage/package.go @@ -95,7 +95,6 @@ func CustomizePackaging(pkgFlavor PackagingFlavor) { args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfig) sampleRulesTarget = "/etc/{{.BeatName}}/" + defaultSampleRulesTarget case devtools.Docker: - args.Spec.ExtraVar("user", "root") default: panic(errors.Errorf("unhandled package type: %v", pkgType)) } diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 1cfd2402193..dbfbc9f4b7a 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -340,7 +340,7 @@ shared: buildFrom: 'centos:7' dockerfile: 'Dockerfile.elastic-agent.tmpl' docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl' - user: 'root' + user: '{{ .BeatName }}' linux_capabilities: '' files: 'elastic-agent.yml': diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 5e6c0fcd6cd..7ab87f6f3ec 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -12,6 +12,8 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s chown -R root:root {{ $beatHome }} && \ find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + find {{ $beatHome }}/data -type d -exec chmod 0770 {} \; && \ + find {{ $beatHome }}/data -type f -exec chmod 0660 {} \; && \ rm {{ $beatBinary }} && \ ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \ chmod 0750 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \ @@ -21,7 +23,7 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s {{- range $i, $modulesd := .ModulesDirs }} chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ {{- end }} - chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs + true FROM {{ .from }} @@ -69,6 +71,10 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint COPY --from=home {{ $beatHome }} {{ $beatHome }} +# Elastic Agent needs group permissions in the home itself to be able to +# create fleet.yml when running as non-root. +RUN chmod 0770 {{ $beatHome }} + RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 0f8b44b0750..62dd7916437 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -1,8 +1,10 @@ -import os +import docker import filebeat +import os import unittest from beat.beat import INTEGRATION_TESTS +from contextlib import contextmanager class TestAutodiscover(filebeat.BaseTest): @@ -16,47 +18,30 @@ def test_docker(self): """ Test docker autodiscover starts input """ - import docker - docker_client = docker.from_env() - - self.render_config_template( - inputs=False, - autodiscover={ - 'docker': { - 'cleanup_timeout': '0s', - 'templates': ''' - - condition: - equals.docker.container.image: busybox - config: - - type: log - paths: - - %s/${data.docker.container.image}.log - ''' % self.working_dir, + with self.container_running() as container: + self.render_config_template( + inputs=False, + autodiscover={ + 'docker': { + 'cleanup_timeout': '0s', + 'templates': f''' + - condition: + equals.docker.container.name: {container.name} + config: + - type: log + paths: + - %s/${{data.docker.container.name}}.log + ''' % self.working_dir, + }, }, - }, - ) + ) - with open(os.path.join(self.working_dir, 'busybox.log'), 'wb') as f: - f.write(b'Busybox output 1\n') - - proc = self.start_beat() - docker_client.images.pull('busybox') - docker_client.containers.run('busybox', 'sleep 1') + proc = self.start_beat() + self._test(container) - self.wait_until(lambda: self.log_contains('Starting runner: input')) self.wait_until(lambda: self.log_contains('Stopping runner: input')) - - output = self.read_output_json() proc.check_kill_and_wait() - # Check metadata is added - assert output[0]['message'] == 'Busybox output 1' - assert output[0]['container']['image']['name'] == 'busybox' - assert output[0]['docker']['container']['labels'] == {} - assert 'name' in output[0]['container'] - - self.assert_fields_are_documented(output[0]) - @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") @@ -64,41 +49,47 @@ def test_default_settings(self): """ Test docker autodiscover default config settings """ - import docker - docker_client = docker.from_env() - - self.render_config_template( - inputs=False, - autodiscover={ - 'docker': { - 'cleanup_timeout': '0s', - 'hints.enabled': 'true', - 'hints.default_config': ''' - type: log - paths: - - %s/${data.container.image}.log - ''' % self.working_dir, + with self.container_running() as container: + self.render_config_template( + inputs=False, + autodiscover={ + 'docker': { + 'cleanup_timeout': '0s', + 'hints.enabled': 'true', + 'hints.default_config': ''' + type: log + paths: + - %s/${data.container.name}.log + ''' % self.working_dir, + }, }, - }, - ) + ) + proc = self.start_beat() + self._test(container) - with open(os.path.join(self.working_dir, 'busybox.log'), 'wb') as f: - f.write(b'Busybox output 1\n') + self.wait_until(lambda: self.log_contains('Stopping runner: input')) + proc.check_kill_and_wait() - proc = self.start_beat() - docker_client.images.pull('busybox') - docker_client.containers.run('busybox', 'sleep 1') + def _test(self, container): + with open(os.path.join(self.working_dir, f'{container.name}.log'), 'wb') as f: + f.write(b'Busybox output 1\n') self.wait_until(lambda: self.log_contains('Starting runner: input')) - self.wait_until(lambda: self.log_contains('Stopping runner: input')) + self.wait_until(lambda: self.output_has(lines=1)) output = self.read_output_json() - proc.check_kill_and_wait() # Check metadata is added assert output[0]['message'] == 'Busybox output 1' - assert output[0]['container']['image']['name'] == 'busybox' - assert output[0]['docker']['container']['labels'] == {} + assert output[0]['container']['name'] == container.name + assert output[0]['docker']['container']['labels'] == container.labels assert 'name' in output[0]['container'] self.assert_fields_are_documented(output[0]) + + @contextmanager + def container_running(self, image_name='busybox:latest'): + docker_client = docker.from_env() + container = docker_client.containers.run(image_name, 'sleep 60', detach=True, remove=True) + yield container + container.remove(force=True) diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 8353b4cc22f..1d593feb468 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -25,4 +25,4 @@ stages: parameters: - "armTest" unitTest: - mage: "mage build unitTest" \ No newline at end of file + mage: "mage build unitTest" diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 989f8e6d7b6..7484e6465b7 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -84,7 +84,7 @@ func Package() { // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages(devtools.WithRootUserContainer()) + return devtools.TestPackages() } // Update is an alias for running fields, dashboards, config. diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index d9475d35be3..c466d0c656d 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -7,7 +7,11 @@ ==== Breaking changes +- Docker container is not run as root by default. {pull}21213[21213] + ==== Bugfixes +- Copy Action store on upgrade {pull}21298[21298] +- Include inputs in action store actions {pull}21298[21298] ==== New features diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index ec6e76a0995..7296e8189be 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -81,6 +81,9 @@ type Format mg.Namespace // Demo runs agent out of container. type Demo mg.Namespace +// Dev runs package and build for dev purposes. +type Dev mg.Namespace + // Env returns information about the environment. func (Prepare) Env() { mg.Deps(Mkdir("build"), Build.GenerateConfig) @@ -88,6 +91,26 @@ func (Prepare) Env() { RunGo("env") } +// Build builds the agent binary with DEV flag set. +func (Dev) Build() { + dev := os.Getenv(devEnv) + defer os.Setenv(devEnv, dev) + + os.Setenv(devEnv, "true") + devtools.DevBuild = true + mg.Deps(Build.All) +} + +// Package packages the agent binary with DEV flag set. +func (Dev) Package() { + dev := os.Getenv(devEnv) + defer os.Setenv(devEnv, dev) + + os.Setenv(devEnv, "true") + devtools.DevBuild = true + Package() +} + // InstallGoLicenser install go-licenser to check license of the files. func (Prepare) InstallGoLicenser() error { return GoGet(goLicenserRepo) @@ -313,7 +336,7 @@ func requiredPackagesPresent(basePath, beat, version string, requiredPackages [] // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return devtools.TestPackages(devtools.WithRootUserContainer()) + return devtools.TestPackages() } // RunGo runs go command and output the feedback to the stdout and the stderr. diff --git a/x-pack/elastic-agent/pkg/agent/application/config.go b/x-pack/elastic-agent/pkg/agent/application/config.go index ff15ca44074..e42f3dcab28 100644 --- a/x-pack/elastic-agent/pkg/agent/application/config.go +++ b/x-pack/elastic-agent/pkg/agent/application/config.go @@ -11,6 +11,7 @@ import ( "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -54,7 +55,10 @@ func LoadConfigFromFile(path string) (*config.Config, error) { // // This must be used to load the Agent configuration, so that variables defined in the inputs are not // parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler. -func LoadConfig(m map[string]interface{}) (*config.Config, error) { +func LoadConfig(in map[string]interface{}) (*config.Config, error) { + // make copy of a map so we dont affect a caller + m := common.MapStr(in).Clone() + inputs, ok := m["inputs"] if ok { // remove the inputs diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go index cc27846051f..08c38aba8c5 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go @@ -14,6 +14,7 @@ import ( "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" @@ -78,6 +79,10 @@ func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error return errors.New("upgrading to same version") } + if err := copyActionStore(newHash); err != nil { + return errors.New(err, "failed to copy action store") + } + if err := u.changeSymlink(ctx, newHash); err != nil { rollbackInstall(newHash) return err @@ -137,3 +142,21 @@ func isSubdir(base, target string) (bool, error) { func rollbackInstall(hash string) { os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash))) } + +func copyActionStore(newHash string) error { + currentActionStorePath := info.AgentActionStoreFile() + + newHome := filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, newHash)) + newActionStorePath := filepath.Join(newHome, filepath.Base(currentActionStorePath)) + + currentActionStore, err := ioutil.ReadFile(currentActionStorePath) + if os.IsNotExist(err) { + // nothing to copy + return nil + } + if err != nil { + return err + } + + return ioutil.WriteFile(newActionStorePath, currentActionStore, 0600) +} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index a7b56a664ba..77beeb6fe1a 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) func newRunCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command { @@ -82,6 +83,10 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se return err } + if allowEmptyPgp, _ := release.PGP(); allowEmptyPgp { + logger.Warn("Artifact has been build with security disabled. Elastic Agent will not verify signatures of used artifacts.") + } + execPath, err := os.Executable() if err != nil { return err