From b232e989fb7eb0bbe8a6be7ceb1916e9d9baa1fe Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 14 May 2024 13:26:27 +0300 Subject: [PATCH 1/2] [BK] - Remove certain steps from running for Branches (#39533) This commit makes some adjustments in the step conditions after reviewing the Jenkins files and how we perform the step validations in Jenkinsfile. If there is a key `when` inside the Jenkinsfile step, it will validate the required conditions, and if there are no `branches: true`, then it will not include that step. Signed-off-by: Alexandros Sapranidis (cherry picked from commit 176e9b8c6dc55ef211f39f194d1366b593793691) # Conflicts: # .buildkite/packetbeat/pipeline.packetbeat.yml --- .buildkite/auditbeat/auditbeat-pipeline.yml | 8 ++-- .buildkite/filebeat/filebeat-pipeline.yml | 2 +- .buildkite/metricbeat/pipeline.yml | 2 +- .buildkite/packetbeat/pipeline.packetbeat.yml | 44 ++++++++++++++++++- .../x-pack/pipeline.xpack.auditbeat.yml | 2 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 6 +-- .../x-pack/pipeline.xpack.heartbeat.yml | 2 +- .../x-pack/pipeline.xpack.metricbeat.yml | 8 ++-- .../x-pack/pipeline.xpack.packetbeat.yml | 2 +- 9 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 9a28c40b0dc..3fcab1696b4 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -128,7 +128,7 @@ steps: steps: - label: ":linux: Auditbeat Ubuntu Integration Tests" key: "auditbeat-extended-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat @@ -149,7 +149,7 @@ steps: - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat @@ -190,7 +190,7 @@ steps: context: "auditbeat: Linux arm64 Unit Tests" - label: ":mac: Auditbeat macOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -210,7 +210,7 @@ steps: context: "auditbeat: macOS x86_64 Unit Tests" - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 70df2d140b6..c7fbdad80f8 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -148,7 +148,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 405a1983b07..e535038a219 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -241,7 +241,7 @@ steps: - group: "Metricbeat Extended MacOS Tests" key: "metricbeat-extended-macos-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS x64_64 Unit Tests" key: "extended-macos-x64-64-unit-tests" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index c59d8649370..ebfb0de46e9 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -170,7 +170,7 @@ steps: steps: - label: ":mac: MacOS x86_64 Unit Tests" key: "macos-x86-64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -189,6 +189,48 @@ steps: - github_commit_status: context: "packetbeat: Extended MacOS x86_64 Unit Tests" +<<<<<<< HEAD +======= + - label: ":mac: MacOS arm64 Unit Tests" + key: "macos-arm64-unit-tests-extended" + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd packetbeat + mage build unitTest + retry: + automatic: + - limit: 3 + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended MacOS arm64 Unit Tests" + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "linux-arm64-unit-tests-extended" + command: "cd packetbeat && mage build unitTest" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + retry: + automatic: + - limit: 3 + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended Ubuntu ARM Unit Tests" + +>>>>>>> 176e9b8c6d ([BK] - Remove certain steps from running for Branches (#39533)) - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 40ba75998c7..900e8f02abb 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -180,7 +180,7 @@ steps: - group: "x-pack/auditbeat MacOS Extended Tests" key: "x-pack-auditbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" command: | diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 535afd3a768..c62707926bb 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -222,7 +222,7 @@ steps: key: "x-pack-filebeat-extended-tests" steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -242,7 +242,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -262,7 +262,7 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 9c77c101b6a..0e97226aceb 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -149,7 +149,7 @@ steps: - group: "x-pack/heartbeat macOS Extended Tests" key: "x-pack-heartbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: x-pack/heartbeat macOS x86_64 Unit Tests" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 720a6f86d0b..46889eb73ec 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -204,7 +204,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -224,7 +224,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -245,7 +245,7 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" skip: "doesn't belong in a stage in Jenkins, thus skipped" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 command: | @@ -278,7 +278,7 @@ steps: skip: "https://github.com/elastic/beats/issues/36425" # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 6b6dccfe08b..3aa05eff598 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -128,7 +128,7 @@ steps: - group: "x-pack/packetbeat MacOS Extended Tests" key: "x-pack-packetbeat-extended-macos-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" From ac03d1c97d25124d6ee527ac803aec0477f36b6e Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 15 May 2024 14:43:27 +0300 Subject: [PATCH 2/2] Resolve conflicts with 7.17 Signed-off-by: Alexandros Sapranidis --- .buildkite/packetbeat/pipeline.packetbeat.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index ebfb0de46e9..86016e13d78 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -189,8 +189,6 @@ steps: - github_commit_status: context: "packetbeat: Extended MacOS x86_64 Unit Tests" -<<<<<<< HEAD -======= - label: ":mac: MacOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ @@ -230,7 +228,6 @@ steps: - github_commit_status: context: "packetbeat: Extended Ubuntu ARM Unit Tests" ->>>>>>> 176e9b8c6d ([BK] - Remove certain steps from running for Branches (#39533)) - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests