From 2b5dd7bf28cd79e8fcd30cf13cfcbc679b9e046f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 12:55:18 +0200 Subject: [PATCH 01/78] migrate xpack-metricbeat --- .buildkite/pipeline.yml | 55 +++++- .buildkite/scripts/common.sh | 18 ++ .../scripts/generate_packetbeat_pipeline.sh | 2 +- .../generate_xpack_metricbeat_pipeline.sh | 182 ++++++++++++++++++ .buildkite/x-pack/pipeline.xpack.libbeat.yml | 8 +- .../x-pack/pipeline.xpack.metricbeat.yml | 58 +++++- 6 files changed, 314 insertions(+), 9 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34321b61161..e07cfa87fe0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,56 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" - command: echo "Hello!" + - trigger: "beats-packetbeat" + label: ":esbuild: Downstream - beats-packetbeat" + key: "downstream-beats-packetbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-libbeat" + label: ":esbuild: Downstream - beats-libbeat" + key: "downstream-beats-libbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-metricbeat" + label: ":esbuild: Downstream - beats-metricbeat" + key: "downstream-beats-metricbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-winlogbeat" + label: ":esbuild: Downstream - beats-winlogbeat" + key: "downstream-beats-winlogbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-xpack-libbeat" + label: ":esbuild: Downstream - beats-xpack-libbeat" + key: "downstream-beats-xpack-libbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-xpack-metricbeat" + label: ":esbuild: Downstream - beats-xpack-metricbeat" + key: "downstream-beats-xpack-metricbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b797ec718aa..84673866217 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -17,6 +17,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" +[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" +[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" +[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" metricbeat_changeset=( "^metricbeat/.*" @@ -89,6 +92,7 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" @@ -101,6 +105,7 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_PROJECT_NAME" @@ -113,8 +118,10 @@ check_and_set_beat_vars() { BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" + BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" BAETS_GH_MACOS_LABEL="macOS" BAETS_GH_ARM_LABEL="arm" + BAETS_GH_AWS_LABEL="aws" } with_docker_compose() { @@ -320,6 +327,17 @@ are_conditions_met_macos_tests() { return 1 } +re_conditions_met_aws_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_AWS_LABEL}" || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + fi + fi + return 1 +} + are_conditions_met_packaging() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 89ea7a33e20..b35554ce669 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -92,7 +92,7 @@ else exit 0 fi -if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh new file mode 100755 index 00000000000..fe5b5a8ca49 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-metricbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_cloud_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Cloud Tests" + key: "extended-cloud-test" + command: "cd $BEATS_PROJECT_NAME && mage build test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 01695fa4fb6..5da3b6a3ac0 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -20,7 +20,7 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - run_xpack_libbeat" + - select: "Libbeat - run_xpack_libbeat" key: "run_xpack_libbeat" options: - label: "True" @@ -28,7 +28,7 @@ steps: - label: "False" value: "false" default: "false" - - select: "Packetbeat - run_xpack_libbeat_arm_tests" + - select: "Libbeat - run_xpack_libbeat_arm_tests" key: "run_xpack_libbeat_arm_tests" options: - label: "True" @@ -42,8 +42,8 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic packetbeat pipeline" - key: "packetbeat-pipeline" + - label: ":linux: Load dynamic libbeat pipeline" + key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: - github_commit_status: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 34321b61161..8f7b19ec3aa 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -1,5 +1,59 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-metricbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch6" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Metricbeat - run_xpack_metricbeat" + key: "run_xpack_metricbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_macos_tests" + key: "run_xpack_metricbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_aws_tests" + key: "run_xpack_metricbeat_aws_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic metricbeat pipeline" + key: "metricbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From cf44101119051a59f2e7810dd1dde4debab472ac Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 13:29:47 +0200 Subject: [PATCH 02/78] change pre-command --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0ac7c51099c..1096f7c3433 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" From e432525e19726ab2466beda5a05285eb9477a04a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:41:10 +0200 Subject: [PATCH 03/78] add mage_version --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/scripts/setenv.sh | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fe5b5a8ca49..bb3d1ab91a9 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage build test" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 25121de212f..921e8ee5f98 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -8,6 +8,7 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) +MAGE_VERSION="1.15.0" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From 0783eee4bfb56a43b35336a81af6a24ac1c5ec73 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:45:58 +0200 Subject: [PATCH 04/78] add mage_version --- .buildkite/scripts/setenv.sh | 1 + .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 921e8ee5f98..7464e30d86d 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -16,6 +16,7 @@ export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION +export MAGE_VERSION exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 5da3b6a3ac0..cf0494ffef8 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -42,7 +42,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic libbeat pipeline" + - label: ":linux: Load dynamic **x-pack libbeat** pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 8f7b19ec3aa..88695cb176c 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -51,7 +51,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic metricbeat pipeline" + - label: ":linux: Load dynamic **x-pack metricbeat** pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: From 2a1de268b0a7cadefdadef6431c16bcadc213995 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:46:29 +0200 Subject: [PATCH 05/78] disable some trigger steps --- .buildkite/pipeline.yml | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e07cfa87fe0..c28e8d09bf3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,41 +1,41 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - trigger: "beats-packetbeat" - label: ":esbuild: Downstream - beats-packetbeat" - key: "downstream-beats-packetbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-packetbeat" + # label: ":esbuild: Downstream - beats-packetbeat" + # key: "downstream-beats-packetbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-libbeat" - label: ":esbuild: Downstream - beats-libbeat" - key: "downstream-beats-libbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-libbeat" + # label: ":esbuild: Downstream - beats-libbeat" + # key: "downstream-beats-libbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-metricbeat" - label: ":esbuild: Downstream - beats-metricbeat" - key: "downstream-beats-metricbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-metricbeat" + # label: ":esbuild: Downstream - beats-metricbeat" + # key: "downstream-beats-metricbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-winlogbeat" - label: ":esbuild: Downstream - beats-winlogbeat" - key: "downstream-beats-winlogbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-winlogbeat" + # label: ":esbuild: Downstream - beats-winlogbeat" + # key: "downstream-beats-winlogbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - trigger: "beats-xpack-libbeat" label: ":esbuild: Downstream - beats-xpack-libbeat" From 7e55863013e3096dad24337d4b63d1960cab5080 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:59:06 +0200 Subject: [PATCH 06/78] change command with mage --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bb3d1ab91a9..7396d7e88c1 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build test" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index cf0494ffef8..0c745b1a09d 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -42,7 +42,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic **x-pack libbeat** pipeline" + - label: ":linux: Load dynamic x-pack libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 88695cb176c..a2ca41a620f 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -51,7 +51,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic **x-pack metricbeat** pipeline" + - label: ":linux: Load dynamic x-pack metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: From cf55e5bf32e2836127e1e2f2da936ddae8a5df29 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 15:21:07 +0200 Subject: [PATCH 07/78] add ASDF_MAGE_VERSION --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/scripts/setenv.sh | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 7396d7e88c1..d607572b946 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build test" + command: "cd $BEATS_PROJECT_NAME && mage build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7464e30d86d..eb374e71c87 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -8,7 +8,7 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) -MAGE_VERSION="1.15.0" +ASDF_MAGE_VERSION="1.15.0" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -16,7 +16,7 @@ export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION -export MAGE_VERSION +export ASDF_MAGE_VERSION exportVars() { local platform_type="$(uname)" From def5673254040db0e8194900b8722dc231da853d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 16:21:29 +0200 Subject: [PATCH 08/78] fix arm image name --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index a2ca41a620f..eca0442fb11 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -3,7 +3,7 @@ name: "beats-xpack-metricbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch6" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/general-windows-10" IMAGE_WIN_11: "family/general-windows-11" IMAGE_WIN_2016: "family/core-windows-2016" From 85bfec853512915c6a04dae6f8b68efe71ede6ef Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 17:11:32 +0200 Subject: [PATCH 09/78] fix macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index d607572b946..fe5b5a8ca49 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: ".buildkite/scripts/unit_tests.sh" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" From 5af9510b6a9eda5b4133f20e5edee0544519059a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 10:58:22 +0200 Subject: [PATCH 10/78] test arm packaging --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fe5b5a8ca49..790e73c3bb3 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -159,7 +159,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - label: ":linux: Packaging ARM" key: "packaging-arm" From 90bec63d1cf8d527b184e5ebac30230f7d7b4e84 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 12:22:16 +0200 Subject: [PATCH 11/78] test aws/macos --- .buildkite/scripts/common.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 84673866217..44cfab1a37b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -308,7 +308,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -318,8 +318,8 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -330,7 +330,7 @@ are_conditions_met_macos_tests() { re_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_AWS_LABEL}" || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi From bd038c2144983f46a22c3a31453814eb2fca51db Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 14:50:59 +0200 Subject: [PATCH 12/78] test macos --- .buildkite/scripts/generate_metricbeat_pipeline.sh | 8 ++++++-- .buildkite/scripts/generate_packetbeat_pipeline.sh | 4 ++-- .buildkite/scripts/generate_winlogbeat_pipeline.sh | 2 +- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 7 +++++-- .buildkite/scripts/setenv.sh | 4 ++++ .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index e91896eb70c..15d0a737523 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -111,6 +111,10 @@ else exit 0 fi +#---------------debug--------------- +echo "${GITHUB_PR_LABELS}" +#------------end of debug----------- + echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML @@ -149,7 +153,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -159,7 +163,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index b35554ce669..97bdb531a6b 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -150,7 +150,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -160,7 +160,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index 1eb1b459c92..ce812016e79 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -96,7 +96,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" YAML diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 790e73c3bb3..e1cf6d3f277 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -110,6 +110,9 @@ if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then YAML fi +#---------------debug--------------- +echo "${GITHUB_PR_LABELS}" +#------------end of debug----------- if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML @@ -159,7 +162,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -169,7 +172,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index eb374e71c87..7bc0cfd1694 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -9,6 +9,8 @@ SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" +PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +PACKAGING_ARM_PLATFORMS="linux/arm64" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -17,6 +19,8 @@ export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION export ASDF_MAGE_VERSION +export PACKAGING_PLATFORMS +export PACKAGING_ARM_PLATFORMS exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index eca0442fb11..17529a33858 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -18,7 +18,7 @@ env: steps: - input: "Input Parameters" - key: "input-run-all-stages" + key: "force-run-stages" fields: - select: "Metricbeat - run_xpack_metricbeat" key: "run_xpack_metricbeat" From 77b29b26983e23b68ac26db40af10af142e4b5c9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 15:23:48 +0200 Subject: [PATCH 13/78] test macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index e1cf6d3f277..c501c9eecad 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -101,7 +101,7 @@ else exit 0 fi -if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then +if are_conditions_met_aws_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" From 5b41d3b3e99be1b333cd429b5daac87c60b75ebf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 15:29:11 +0200 Subject: [PATCH 14/78] test macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c501c9eecad..bc1bde53a4e 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -128,7 +128,7 @@ if are_conditions_met_macos_tests; then YAML fi -if are_conditions_met_cloud_tests; then +if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" From 2751e38c5a3ce2556c34930dd019b754621ae9d6 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:03:36 +0200 Subject: [PATCH 15/78] test macos --- .buildkite/pipeline.yml | 32 +++++++++---------- .buildkite/scripts/common.sh | 2 +- .../generate_xpack_metricbeat_pipeline.sh | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c28e8d09bf3..1bd21ef6acb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -37,20 +37,20 @@ steps: # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-xpack-libbeat" - label: ":esbuild: Downstream - beats-xpack-libbeat" - key: "downstream-beats-xpack-libbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-xpack-libbeat" + # label: ":esbuild: Downstream - beats-xpack-libbeat" + # key: "downstream-beats-xpack-libbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-xpack-metricbeat" - label: ":esbuild: Downstream - beats-xpack-metricbeat" - key: "downstream-beats-xpack-metricbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-xpack-metricbeat" + # label: ":esbuild: Downstream - beats-xpack-metricbeat" + # key: "downstream-beats-xpack-metricbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 44cfab1a37b..ee8cef67017 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -327,7 +327,7 @@ are_conditions_met_macos_tests() { return 1 } -re_conditions_met_aws_tests() { +are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bc1bde53a4e..a640349ae38 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -101,7 +101,7 @@ else exit 0 fi -if are_conditions_met_aws_tests || are_conditions_met_macos_tests; then +if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then cat >> $pipelineName <<- YAML - group: "Extended Tests" From 88800c82d858220f95fb87f5916cb75847d2a92b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:09:16 +0200 Subject: [PATCH 16/78] test macos --- .buildkite/pipeline.yml | 4 ++++ .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1bd21ef6acb..c5dc4a978a2 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: + + - label: "Example test" + command: echo "Hello!" + # - trigger: "beats-packetbeat" # label: ":esbuild: Downstream - beats-packetbeat" # key: "downstream-beats-packetbeat" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 17529a33858..83a7fe886c8 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -9,6 +9,7 @@ env: IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" From 4805ccb6a20ae680b0643381dea3345d2ed26e67 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:29:48 +0200 Subject: [PATCH 17/78] fix macos declare --- .buildkite/scripts/common.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ee8cef67017..b33368f184c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -84,6 +84,32 @@ packaging_changeset=( ".go-version" ) +case "${BUILDKITE_PIPELINE_SLUG}" in + "beats-metricbeat") + BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]} + ;; + "beats-libbeat") + BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} + ;; + "beats-packetbeat") + BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} + ;; + "beats-winlogbeat") + BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} + ;; + "beats-xpack-libbeat") + BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} + ;; + "beats-xpack-metricbeat") + BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} + ;; + *) + echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + ;; +esac + + + check_and_set_beat_vars() { if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - @@ -94,7 +120,6 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -107,7 +132,6 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" From e2425682f71c6cc5606358b3c22c2646cf5bb9ab Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 09:34:42 +0200 Subject: [PATCH 18/78] add export MODULE --- .buildkite/scripts/common.sh | 18 +++++++++++++++--- .../scripts/generate_metricbeat_pipeline.sh | 4 ---- .../generate_xpack_metricbeat_pipeline.sh | 3 --- .buildkite/scripts/setenv.sh | 10 +++++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b33368f184c..870677f07c2 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -379,18 +379,30 @@ config_git() { } if ! are_changed_only_paths "${docs_changeset[@]}" ; then - ONLY_DOCS="false" + export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." else echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." fi if are_paths_changed "${go_mod_changeset[@]}" ; then - GO_MOD_CHANGES="true" + export GO_MOD_CHANGES="true" fi if are_paths_changed "${packaging_changeset[@]}" ; then - PACKAGING_CHANGES="true" + export PACKAGING_CHANGES="true" +fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") + AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then + export MODULE="true" + else + export MODULE="false" + fi + + fi check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 15d0a737523..477f8fb25a3 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -111,10 +111,6 @@ else exit 0 fi -#---------------debug--------------- -echo "${GITHUB_PR_LABELS}" -#------------end of debug----------- - echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index a640349ae38..06ce06e03d0 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -110,9 +110,6 @@ if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then YAML fi -#---------------debug--------------- -echo "${GITHUB_PR_LABELS}" -#------------end of debug----------- if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7bc0cfd1694..7f455758b64 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,6 +11,8 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -47,8 +49,14 @@ exportVars() { fi } +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + exportVars + export RACE_DETECTOR="true" + export TEST_COVERAGE="true" + export DOCKER_PULL="0" +fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" From d41b68d929e5bc245664cb86937fd9b92918945c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 12:42:06 +0200 Subject: [PATCH 19/78] test aws --- .buildkite/scripts/common.sh | 9 +- .../generate_xpack_metricbeat_pipeline.sh | 126 +++++++++--------- 2 files changed, 68 insertions(+), 67 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 870677f07c2..e7ef3dae2d0 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -395,13 +395,14 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + AWS_MODULE_PATH_TRNSFORMED=$(echo "$directory" | sed 's/\//\\\//g') + AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then - export MODULE="true" + MODULE="aws" else - export MODULE="false" + MODULE="" fi - + export MODULE fi diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 06ce06e03d0..2bcdf8cebfd 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -59,41 +59,41 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - group: "Extended Windows Tests" + # key: "extended-win-tests" + # steps: + # - label: ":windows: Windows 10 Unit Tests" + # key: "extended-win-10-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Windows 11 Unit Tests" + # key: "extended-win-11-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_11}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Win 2019 Unit Tests" + # key: "extended-win-2019-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2019}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -143,34 +143,34 @@ echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" + # - wait: ~ + # depends_on: + # - step: "mandatory-tests" + # allow_failure: false + + # - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + # key: "packaging" + # steps: + # - label: ":linux: Packaging Linux" + # key: "packaging-linux" + # command: "cd $BEATS_PROJECT_NAME && mage package" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # env: + # PLATFORMS: "${PACKAGING_PLATFORMS}" + + # - label: ":linux: Packaging ARM" + # key: "packaging-arm" + # command: "cd $BEATS_PROJECT_NAME && mage package" + # agents: + # provider: "aws" + # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + # instanceType: "${AWS_ARM_INSTANCE_TYPE}" + # env: + # PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + # PACKAGES: "docker" YAML fi From 531975317242253a1bf39a017c6ea3db38a9d76e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 12:46:14 +0200 Subject: [PATCH 20/78] test aws --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e7ef3dae2d0..21c48dbb295 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -395,7 +395,7 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_TRNSFORMED=$(echo "$directory" | sed 's/\//\\\//g') + AWS_MODULE_PATH_TRNSFORMED=$(echo "$AWS_MODULE_PATH" | sed 's/\//\\\//g') AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then MODULE="aws" From 32229ce7bd8e2a64388790cc861ffb8561f1b56b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 13:59:34 +0200 Subject: [PATCH 21/78] add withModule --- .buildkite/scripts/common.sh | 33 +++++++++++++------ .../generate_xpack_metricbeat_pipeline.sh | 30 ++++++++--------- .buildkite/scripts/setenv.sh | 2 -- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 21c48dbb295..da3fea4f89e 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,6 +9,8 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -378,6 +380,26 @@ config_git() { fi } +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -394,16 +416,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_TRNSFORMED=$(echo "$AWS_MODULE_PATH" | sed 's/\//\\\//g') - AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then - MODULE="aws" - else - MODULE="" - fi - export MODULE - + withModule "x-pack/metricbeat/module/aws" fi check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 2bcdf8cebfd..19bd87fbb8e 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -42,21 +42,21 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # key: "mandatory-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_2016}" + # - "${IMAGE_WIN_2022}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h # - group: "Extended Windows Tests" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7f455758b64..d85c0e9c0ea 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,8 +11,6 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From a09803e99651ea6ed4e7ce7a6874e7cef219fac4 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:13:15 +0200 Subject: [PATCH 22/78] move withModule to pre-command --- .buildkite/scripts/common.sh | 55 ------------------------------------ .buildkite/scripts/setenv.sh | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index da3fea4f89e..a3e2c7aa557 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -293,37 +293,6 @@ retry() { return 0 } -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then - return 0 - fi - return 1 -} - are_conditions_met_mandatory_tests() { if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 @@ -380,26 +349,6 @@ config_git() { fi } -withModule() { - local module_path=$1 - if [[ "$module_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local module_name="${module_path#*module/}" - local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') - local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" - local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE="${module_name}" - else - MODULE="" - fi - echo "MODULE=$MODULE" - export MODULE -} - if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -415,8 +364,4 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "x-pack/metricbeat/module/aws" -fi - check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index d85c0e9c0ea..dc3eb4c26e8 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -47,6 +47,57 @@ exportVars() { fi } +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + fi + return 1 +} + +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" @@ -60,3 +111,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export TEST_COVERAGE="true" export DOCKER_PULL="0" fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + withModule "x-pack/metricbeat/module/aws" +fi From 271c788e0a165afae1174179c620f0a55887ac78 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:16:50 +0200 Subject: [PATCH 23/78] move withModule to pre-command --- .buildkite/scripts/common.sh | 2 -- .buildkite/scripts/setenv.sh | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a3e2c7aa557..309886d973f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,8 +9,6 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dc3eb4c26e8..119728b1809 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,6 +11,8 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From cd2282de13c1022803a29d9386a413f2def05304 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:24:12 +0200 Subject: [PATCH 24/78] move withModule to command --- .buildkite/scripts/common.sh | 57 ++++++++++++++++++++++++++++++++++++ .buildkite/scripts/setenv.sh | 53 --------------------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 309886d973f..da3fea4f89e 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,6 +9,8 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -291,6 +293,37 @@ retry() { return 0 } +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + fi + return 1 +} + are_conditions_met_mandatory_tests() { if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 @@ -347,6 +380,26 @@ config_git() { fi } +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -362,4 +415,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + withModule "x-pack/metricbeat/module/aws" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 119728b1809..97ca2102c84 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,8 +11,6 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -49,57 +47,6 @@ exportVars() { fi } -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then - return 0 - fi - return 1 -} - -withModule() { - local module_path=$1 - if [[ "$module_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local module_name="${module_path#*module/}" - local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') - local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" - local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE="${module_name}" - else - MODULE="" - fi - echo "MODULE=$MODULE" - export MODULE -} - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" From 8eba9dce5b1f2def25250aeb48a67800fb19e07b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:31:46 +0200 Subject: [PATCH 25/78] fix withModule --- .buildkite/scripts/common.sh | 1 - .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 6 ++++++ .buildkite/scripts/setenv.sh | 4 ---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index da3fea4f89e..2d938dbc6a8 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -397,7 +397,6 @@ withModule() { MODULE="" fi echo "MODULE=$MODULE" - export MODULE } if ! are_changed_only_paths "${docs_changeset[@]}" ; then diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 19bd87fbb8e..40723f63c91 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,6 +27,8 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -36,6 +38,8 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -130,6 +134,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: "cd $BEATS_PROJECT_NAME && mage build test" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 97ca2102c84..d85c0e9c0ea 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -60,7 +60,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export TEST_COVERAGE="true" export DOCKER_PULL="0" fi - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "x-pack/metricbeat/module/aws" -fi From 1781fc004b0fa4ee03b5cfba826794e2f4fc281b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 19:39:33 +0200 Subject: [PATCH 26/78] test aws changes --- .buildkite/scripts/common.sh | 2 +- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2d938dbc6a8..7855a84a991 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -393,7 +393,7 @@ withModule() { local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then MODULE="${module_name}" - else + elif [ -d "${module_path}" ]; then MODULE="" fi echo "MODULE=$MODULE" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 40723f63c91..fc506b69d8a 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -139,7 +139,7 @@ if are_conditions_met_aws_tests; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml index 5401ba81f41..262bf7bf87b 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml @@ -9,6 +9,7 @@ input: name: - BucketSizeBytes - NumberOfObjects + - TestName processors: - rename: From c12fe5ab52774bec32fb73597f8ef3b13a51a755 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 19:44:13 +0200 Subject: [PATCH 27/78] test aws changes --- x-pack/metricbeat/module/aws/variables.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index e4b95d471b2..ea5ecaea96b 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,3 +20,7 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } + +variable "TESTVAR" { + default = "unknown-repo-name" +} From dd1e93ebe8bc6eb2d9056dfbf4c99813c066ac43 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 24 Feb 2024 12:06:05 +0200 Subject: [PATCH 28/78] change the are_changed_only_paths --- .buildkite/scripts/common.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7855a84a991..b5e57e1b548 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -316,9 +316,15 @@ are_paths_changed() { are_changed_only_paths() { local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + local changed_files=($(git diff --name-only HEAD@{1} HEAD)) + local matched_files=() + for pattern in "${patterns[@]}"; do + local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) + if [ "${#matched[@]}" -gt 0 ]; then + matched_files+=("${matched[@]}") + fi + done + if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then return 0 fi return 1 From d0f433b9708261bcf6599e94310a3f2ee5949eed Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 24 Feb 2024 12:35:07 +0200 Subject: [PATCH 29/78] revert aws changes and test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml | 1 - x-pack/metricbeat/module/aws/variables.tf | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fc506b69d8a..76352b206b6 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -131,7 +131,7 @@ fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" + - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" command: "cd $BEATS_PROJECT_NAME && mage build test" env: diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml index 262bf7bf87b..5401ba81f41 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml @@ -9,7 +9,6 @@ input: name: - BucketSizeBytes - NumberOfObjects - - TestName processors: - rename: diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index ea5ecaea96b..e4b95d471b2 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,7 +20,3 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } - -variable "TESTVAR" { - default = "unknown-repo-name" -} From 5696ea284baff2351a03678b05b4cec45dfd9ebf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 13:42:27 +0200 Subject: [PATCH 30/78] test with 240min timeout --- .buildkite/scripts/common.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b5e57e1b548..073997ab3f0 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -110,8 +110,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in ;; esac - - check_and_set_beat_vars() { if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - From ba45c923f952cc93866d0d50f2ceb0f6cce09eb8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 13:51:26 +0200 Subject: [PATCH 31/78] revert aws changes and test --- .buildkite/pipeline.yml | 55 ----- .../generate_xpack_metricbeat_pipeline.sh | 190 +++++++++--------- 2 files changed, 97 insertions(+), 148 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c5dc4a978a2..34321b61161 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,60 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" command: echo "Hello!" - - # - trigger: "beats-packetbeat" - # label: ":esbuild: Downstream - beats-packetbeat" - # key: "downstream-beats-packetbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-libbeat" - # label: ":esbuild: Downstream - beats-libbeat" - # key: "downstream-beats-libbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-metricbeat" - # label: ":esbuild: Downstream - beats-metricbeat" - # key: "downstream-beats-metricbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-winlogbeat" - # label: ":esbuild: Downstream - beats-winlogbeat" - # key: "downstream-beats-winlogbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-xpack-libbeat" - # label: ":esbuild: Downstream - beats-xpack-libbeat" - # key: "downstream-beats-xpack-libbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-xpack-metricbeat" - # label: ":esbuild: Downstream - beats-xpack-metricbeat" - # key: "downstream-beats-xpack-metricbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 76352b206b6..2d49a0137b5 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -46,58 +46,58 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # key: "mandatory-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_2016}" - # - "${IMAGE_WIN_2022}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - # - group: "Extended Windows Tests" - # key: "extended-win-tests" - # steps: - # - label: ":windows: Windows 10 Unit Tests" - # key: "extended-win-10-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Windows 11 Unit Tests" - # key: "extended-win-11-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_11}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Win 2019 Unit Tests" - # key: "extended-win-2019-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2019}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -105,7 +105,9 @@ else exit 0 fi -if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +#TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved +# if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +if [[ are_conditions_met_macos_tests ]]; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -129,54 +131,56 @@ if are_conditions_met_macos_tests; then YAML fi -if are_conditions_met_aws_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ${MODULE^^} Cloud Tests" - key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage build test" - env: - MODULE: "$MODULE" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +#TODO: uncomment the commented-out below step when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -YAML -fi +# if are_conditions_met_aws_tests; then +# cat >> $pipelineName <<- YAML +# - label: ":linux: ${MODULE^^} Cloud Tests" +# key: "extended-cloud-test" +# command: "cd $BEATS_PROJECT_NAME && mage build test" +# env: +# MODULE: "$MODULE" +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - # - wait: ~ - # depends_on: - # - step: "mandatory-tests" - # allow_failure: false - - # - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - # key: "packaging" - # steps: - # - label: ":linux: Packaging Linux" - # key: "packaging-linux" - # command: "cd $BEATS_PROJECT_NAME && mage package" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # env: - # PLATFORMS: "${PACKAGING_PLATFORMS}" - - # - label: ":linux: Packaging ARM" - # key: "packaging-arm" - # command: "cd $BEATS_PROJECT_NAME && mage package" - # agents: - # provider: "aws" - # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - # instanceType: "${AWS_ARM_INSTANCE_TYPE}" - # env: - # PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - # PACKAGES: "docker" + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" YAML fi From eaaa0955930e792d36416c9b5793cc3063ae0929 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 14:24:49 +0200 Subject: [PATCH 32/78] full test --- .../generate_xpack_metricbeat_pipeline.sh | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 2d49a0137b5..2e54f3f5bbd 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -106,8 +106,8 @@ else fi #TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -# if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then -if [[ are_conditions_met_macos_tests ]]; then +if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +# if [[ are_conditions_met_macos_tests ]]; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -133,21 +133,21 @@ fi #TODO: uncomment the commented-out below step when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -# if are_conditions_met_aws_tests; then -# cat >> $pipelineName <<- YAML -# - label: ":linux: ${MODULE^^} Cloud Tests" -# key: "extended-cloud-test" -# command: "cd $BEATS_PROJECT_NAME && mage build test" -# env: -# MODULE: "$MODULE" -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi +if are_conditions_met_aws_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ${MODULE^^} Cloud Tests" + key: "extended-cloud-test" + command: "cd $BEATS_PROJECT_NAME && mage build test" + env: + MODULE: "$MODULE" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then From 4612e004dc7991645b2862cef1b294f0c699df1f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 08:52:12 +0200 Subject: [PATCH 33/78] change the machine config for aws and go-int test steps --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 2e54f3f5bbd..02444022644 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -33,6 +33,8 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -144,6 +146,8 @@ if are_conditions_met_aws_tests; then provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML From e709ccf277b87b0b60761f229ad1925ece7b7d7c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 19:13:41 +0200 Subject: [PATCH 34/78] add aws and terraform --- .buildkite/hooks/pre-command | 28 +++++++++++++++++++++ .buildkite/scripts/common.sh | 47 +++++++++++++++++++++++++++++++++++- .buildkite/scripts/setenv.sh | 9 ++++++- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1096f7c3433..c20a55a4250 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,6 +2,27 @@ set -euo pipefail +AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -19,3 +40,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S exit 0 fi fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_SECRET_KEY + BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_ACCESS_KEY +fi diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 073997ab3f0..9e575f02844 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -158,6 +158,19 @@ with_docker_compose() { docker-compose version } +with_Terraform() { + echo "Setting up the Terraform environment..." + local path_to_file="${WORKSPACE}/terraform.zip" + create_workspace + check_platform_architeture + retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip" + unzip -q ${path_to_file} -d ${BIN}/ + rm ${path_to_file} + chmod +x ${BIN}/terraform + export PATH="${BIN}:${PATH}" + terraform version +} + create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -403,6 +416,37 @@ withModule() { echo "MODULE=$MODULE" } +terraformInit() { + local dir=$1 + echo "Terraform Init on $BEATS_PROJECT_NAME" + pushd "${BEATS_PROJECT_NAME}" > /dev/null + terraform init + popd > /dev/null +} + +startCloudTestEnv() { + local dir=$1 + withAWS + echo "--- Run docker-compose services for emulated cloud env" + docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + with_Terraform + terraformInit "$dir" + export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') + export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" + export TF_VAR_CREATED_DATE=$(date +%s) + export TF_VAR_ENVIRONMENT="ci" + export TF_VAR_REPO="${REPO}" + pushd "${dir}" > /dev/null + terraform apply -auto-approve + popd > /dev/null +} + +withAWS() { + export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY + export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY + export TEST_TAGS="${TEST_TAGS},aws" +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -419,7 +463,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "x-pack/metricbeat/module/aws" + startCloudTestEnv "${MODULE_DIR}" + withModule "${MODULE_DIR}" fi check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index d85c0e9c0ea..816dd608ce4 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail - +REPO="beats" SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" @@ -9,8 +9,10 @@ SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" +ASDF_TERRAFORM_VERSION="1.0.2" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +AWS_REGION="eu-central-1" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -21,6 +23,9 @@ export GO_VERSION export ASDF_MAGE_VERSION export PACKAGING_PLATFORMS export PACKAGING_ARM_PLATFORMS +export REPO +export ASDF_TERRAFORM_VERSION +export AWS_REGION exportVars() { local platform_type="$(uname)" @@ -52,6 +57,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" + export TEST_TAGS="oracle" fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then @@ -59,4 +65,5 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" + export MODULE_DIR="x-pack/metricbeat/module/aws" fi From 79c8f959c8c6721186b2947ec3b088fd3251afc6 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 19:22:57 +0200 Subject: [PATCH 35/78] add docker-compose --- .buildkite/scripts/common.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9e575f02844..68f45ed0ffc 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -463,6 +463,20 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if command -v docker-compose &> /dev/null + then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $4}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then + echo "Versions match. No need to install docker-compose. Exiting." + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "aarch64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION_AARCH64}" + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "x86_64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi + else + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi startCloudTestEnv "${MODULE_DIR}" withModule "${MODULE_DIR}" fi From ef0bd8448b8edd09d46155eb0b9c89ac1b3c52bd Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 29 Feb 2024 10:48:18 +0200 Subject: [PATCH 36/78] add withDocker --- .buildkite/scripts/common.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 68f45ed0ffc..e3abae9b812 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -427,6 +427,7 @@ terraformInit() { startCloudTestEnv() { local dir=$1 withAWS + withDocker echo "--- Run docker-compose services for emulated cloud env" docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK with_Terraform @@ -447,6 +448,11 @@ withAWS() { export TEST_TAGS="${TEST_TAGS},aws" } +withDocker() { + echo "Setting up the Docker environment..." + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." From 873c7d885f7e522238189549b8673fc13c9ac977 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 29 Feb 2024 10:52:41 +0200 Subject: [PATCH 37/78] add withDocker --- .buildkite/hooks/pre-command | 6 ++++++ .buildkite/scripts/common.sh | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c20a55a4250..b1503e29f7d 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -23,6 +23,11 @@ retry() { return 0 } +withDocker() { + echo "Setting up the Docker environment..." + apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -46,4 +51,5 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export BEATS_AWS_SECRET_KEY BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_ACCESS_KEY + withDocker fi diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e3abae9b812..68f45ed0ffc 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -427,7 +427,6 @@ terraformInit() { startCloudTestEnv() { local dir=$1 withAWS - withDocker echo "--- Run docker-compose services for emulated cloud env" docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK with_Terraform @@ -448,11 +447,6 @@ withAWS() { export TEST_TAGS="${TEST_TAGS},aws" } -withDocker() { - echo "Setting up the Docker environment..." - sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -} - if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." From ae65e7f96ef14a495d29040546984dd317fa980b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 29 Feb 2024 12:18:24 +0200 Subject: [PATCH 38/78] add cloud scrit and replace commands in the Cloud and Go-int steps --- .buildkite/hooks/pre-command | 6 ------ .buildkite/scripts/cloud_tests.sh | 12 ++++++++++++ .buildkite/scripts/common.sh | 19 ------------------- .../generate_xpack_metricbeat_pipeline.sh | 8 ++++---- .buildkite/scripts/install_tools.sh | 5 +++++ 5 files changed, 21 insertions(+), 29 deletions(-) create mode 100755 .buildkite/scripts/cloud_tests.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b1503e29f7d..c20a55a4250 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -23,11 +23,6 @@ retry() { return 0 } -withDocker() { - echo "Setting up the Docker environment..." - apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -} - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -51,5 +46,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export BEATS_AWS_SECRET_KEY BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_ACCESS_KEY - withDocker fi diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh new file mode 100755 index 00000000000..df41748aa56 --- /dev/null +++ b/.buildkite/scripts/cloud_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage build test + +popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 68f45ed0ffc..e4eebdf2820 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -462,23 +462,4 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if command -v docker-compose &> /dev/null - then - echo "Found docker-compose. Checking version.." - FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $4}'|sed s/\,//) - if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then - echo "Versions match. No need to install docker-compose. Exiting." - elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "aarch64" ]]; then - with_docker_compose "${DOCKER_COMPOSE_VERSION_AARCH64}" - elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "x86_64" ]]; then - with_docker_compose "${DOCKER_COMPOSE_VERSION}" - fi - else - with_docker_compose "${DOCKER_COMPOSE_VERSION}" - fi - startCloudTestEnv "${MODULE_DIR}" - withModule "${MODULE_DIR}" -fi - check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 02444022644..b1f96ada9e3 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -26,12 +26,12 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: ".buildkite/scripts/go_int_tests.sh" env: MODULE: "$MODULE" agents: provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" + image: "family/core-ubuntu-2204" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" @@ -139,12 +139,12 @@ if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage build test" + command: ".buildkite/scripts/cloud_tests.sh" env: MODULE: "$MODULE" agents: provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" + image: "family/core-ubuntu-2204" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 80e70ae96c5..2b067415b36 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -36,6 +36,11 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + startCloudTestEnv "${MODULE_DIR}" + withModule "${MODULE_DIR}" +fi + with_go "${GO_VERSION}" with_mage with_python From f8a2a091a1bb5fff81330015ccfacc089706cf23 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 29 Feb 2024 12:27:50 +0200 Subject: [PATCH 39/78] add cloud scrit and replace commands in the Cloud and Go-int steps --- .buildkite/scripts/common.sh | 4 ++++ .buildkite/scripts/install_tools.sh | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e4eebdf2820..8209b912eb1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -462,4 +462,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + withModule "${MODULE_DIR}" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 2b067415b36..061c471e872 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -38,7 +38,6 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then startCloudTestEnv "${MODULE_DIR}" - withModule "${MODULE_DIR}" fi with_go "${GO_VERSION}" From 35d89579c39310884a3c8206022e8cf42a5e5fe7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 29 Feb 2024 12:42:45 +0200 Subject: [PATCH 40/78] fix terraformInit --- .buildkite/scripts/common.sh | 4 +- .../generate_xpack_metricbeat_pipeline.sh | 230 +++++++++--------- .buildkite/scripts/install_tools.sh | 2 +- 3 files changed, 118 insertions(+), 118 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 8209b912eb1..62b739853a5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -418,8 +418,8 @@ withModule() { terraformInit() { local dir=$1 - echo "Terraform Init on $BEATS_PROJECT_NAME" - pushd "${BEATS_PROJECT_NAME}" > /dev/null + echo "Terraform Init on $dir" + pushd "${dir}" > /dev/null terraform init popd > /dev/null } diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index b1f96ada9e3..279edea45e9 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -15,14 +15,14 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + # - label: ":linux: Ubuntu Unit Tests" + # key: "mandatory-linux-unit-test" + # command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -37,69 +37,69 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - env: - MODULE: "$MODULE" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":python: Python Integration Tests" + # key: "mandatory-python-int-test" + # command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + # env: + # MODULE: "$MODULE" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # key: "mandatory-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_2016}" + # - "${IMAGE_WIN_2022}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - group: "Extended Windows Tests" + # key: "extended-win-tests" + # steps: + # - label: ":windows: Windows 10 Unit Tests" + # key: "extended-win-10-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Windows 11 Unit Tests" + # key: "extended-win-11-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_11}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Win 2019 Unit Tests" + # key: "extended-win-2019-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2019}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -119,19 +119,19 @@ if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then YAML fi -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML +# if are_conditions_met_macos_tests; then +# cat >> $pipelineName <<- YAML - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +# - label: ":mac: MacOS Unit Tests" +# key: "extended-macos-unit-tests" +# command: ".buildkite/scripts/unit_tests.sh" +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi +# YAML +# fi #TODO: uncomment the commented-out below step when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved @@ -153,41 +153,41 @@ if are_conditions_met_aws_tests; then YAML fi -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - -YAML -fi +# echo "Check and add the Packaging into the pipeline" +# if are_conditions_met_packaging; then +# cat >> $pipelineName <<- YAML + +# - wait: ~ +# depends_on: +# - step: "mandatory-tests" +# allow_failure: false + +# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 +# key: "packaging" +# steps: +# - label: ":linux: Packaging Linux" +# key: "packaging-linux" +# command: "cd $BEATS_PROJECT_NAME && mage package" +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# env: +# PLATFORMS: "${PACKAGING_PLATFORMS}" + +# - label: ":linux: Packaging ARM" +# key: "packaging-arm" +# command: "cd $BEATS_PROJECT_NAME && mage package" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# env: +# PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" +# PACKAGES: "docker" + +# YAML +# fi echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public cat $pipelineName diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 061c471e872..f13d0b3c396 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -36,7 +36,7 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${platform_type}" == "Linux" ]]; then startCloudTestEnv "${MODULE_DIR}" fi From bf8df0d97cb27230f54b28dfcd115bfd9723791f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 4 Mar 2024 17:41:45 +0200 Subject: [PATCH 41/78] test --- .buildkite/scripts/common.sh | 14 ++++++++------ .buildkite/scripts/setenv.sh | 4 ---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 62b739853a5..d7773f2cb93 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -143,9 +143,9 @@ check_and_set_beat_vars() { BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" - BAETS_GH_MACOS_LABEL="macOS" - BAETS_GH_ARM_LABEL="arm" - BAETS_GH_AWS_LABEL="aws" + BEATS_GH_MACOS_LABEL="macOS" + BEATS_GH_ARM_LABEL="arm" + BEATS_GH_AWS_LABEL="aws" } with_docker_compose() { @@ -212,6 +212,8 @@ with_mage() { for pkg in "${install_packages[@]}"; do go install "${pkg}@latest" done + echo "Download modules to local cache" + retry 3 go mod download } with_go() { @@ -351,7 +353,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -362,7 +364,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -373,7 +375,7 @@ are_conditions_met_macos_tests() { are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 816dd608ce4..dab4fd58e03 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -61,9 +61,5 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - exportVars - export RACE_DETECTOR="true" - export TEST_COVERAGE="true" - export DOCKER_PULL="0" export MODULE_DIR="x-pack/metricbeat/module/aws" fi From 0d28c94febd8f1479d72c5cb7bb2fb4d589f6a56 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 5 Mar 2024 08:39:08 +0200 Subject: [PATCH 42/78] test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 279edea45e9..009f28adeed 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -29,6 +29,7 @@ steps: command: ".buildkite/scripts/go_int_tests.sh" env: MODULE: "$MODULE" + withModule: "true" agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -142,6 +143,7 @@ if are_conditions_met_aws_tests; then command: ".buildkite/scripts/cloud_tests.sh" env: MODULE: "$MODULE" + withModule: "true" agents: provider: "gcp" image: "family/core-ubuntu-2204" From 724e1c2379f19d1f1268bac7c637c996da65d107 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 5 Mar 2024 15:37:36 +0200 Subject: [PATCH 43/78] test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 009f28adeed..bafedc42fa3 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -28,8 +28,7 @@ steps: key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" env: - MODULE: "$MODULE" - withModule: "true" + MODULE: "aws" agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -142,8 +141,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: "$MODULE" - withModule: "true" + MODULE: "aws" agents: provider: "gcp" image: "family/core-ubuntu-2204" From 44fae9b3b5105fd265fe99a949cf6d19dc4d3e9f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 5 Mar 2024 16:06:13 +0200 Subject: [PATCH 44/78] changed the logic of cloud and Go int tests --- .buildkite/scripts/cloud_tests.sh | 6 ++++++ .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/scripts/install_tools.sh | 2 +- x-pack/metricbeat/module/aws/variables.tf | 5 +++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index df41748aa56..8e8e80d4497 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -10,3 +10,9 @@ pushd "${BEATS_PROJECT_NAME}" > /dev/null mage build test popd > /dev/null + +echo "Terraform Cleanup" +.ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + +echo "Docker Compose Cleanup" +.ci/scripts/docker-services-cleanup.sh diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bafedc42fa3..c594075bfbb 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -26,15 +26,15 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" env: - MODULE: "aws" + MODULE: "$MODULE" agents: provider: "gcp" - image: "family/core-ubuntu-2204" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" # - label: ":python: Python Integration Tests" @@ -141,7 +141,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: "aws" + MODULE: "$MODULE" agents: provider: "gcp" image: "family/core-ubuntu-2204" diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index f13d0b3c396..aa740c5b7e7 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -36,7 +36,7 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${platform_type}" == "Linux" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then startCloudTestEnv "${MODULE_DIR}" fi diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index e4b95d471b2..79538f50cdc 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,3 +20,8 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } + +variable "TEST_VAR" { + description = "test variable for test aws module changeset" + default = "unknown-var" +} From b488f2532f38278c3522cca09e539969860e7037 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 5 Mar 2024 16:35:01 +0200 Subject: [PATCH 45/78] aws test --- .buildkite/scripts/cloud_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 8e8e80d4497..33b2ac797d9 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -11,8 +11,8 @@ mage build test popd > /dev/null -echo "Terraform Cleanup" +echo "---Terraform Cleanup" .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -echo "Docker Compose Cleanup" -.ci/scripts/docker-services-cleanup.sh +echo "---Docker Compose Cleanup" +docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK From 2e531f89b4d0d75e3a7386a5517db5c2e71ccb28 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 5 Mar 2024 16:56:12 +0200 Subject: [PATCH 46/78] aws test without aws changes --- x-pack/metricbeat/module/aws/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index 79538f50cdc..e4b95d471b2 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,8 +20,3 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } - -variable "TEST_VAR" { - description = "test variable for test aws module changeset" - default = "unknown-var" -} From e440521fd3161b12535e14e78c7d40368a1ad18f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 10:54:00 +0200 Subject: [PATCH 47/78] Jenkins test --- Jenkinsfile.yml | 18 +---- x-pack/metricbeat/Jenkinsfile.yml | 111 ------------------------------ 2 files changed, 1 insertion(+), 128 deletions(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 811b4ae985b..3e2431a7eed 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,22 +1,6 @@ projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" + ## Changeset macros that are defined here and used in each specific 2.0 pipeline. changeset: diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 27574b62ada..6d0d2319db7 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,43 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-22" ## default label for all the stages stages: - unitTest: - mage: "mage build unitTest" - stage: mandatory - goIntegTest: - mage: "mage goIntegTest" - withModule: true - stage: mandatory - pythonIntegTest: - mage: "mage pythonIntegTest" - withModule: true - stage: mandatory - cloud: - cloud: "mage build test" - withModule: true ## run the ITs only if the changeset affects a specific module. - dirs: ## run the cloud tests for the given modules. - - "x-pack/metricbeat/module/aws" - when: ## Override the top-level when. - parameters: - - "awsCloudTests" - comments: - - "/test x-pack/metricbeat for aws cloud" - labels: - - "aws" - # stage: extended - # Skip test until fixed https://github.com/elastic/beats/issues/36425 - #cloudAWS: - # cloud: "mage build test goIntegTest" - # withAWS: true ## Enable the tests to run in AWS - # withModule: true ## run the ITs only if the changeset affects a specific module. - # dirs: ## run the cloud tests for the given modules. - # - "x-pack/metricbeat/module/aws" - # when: ## Override the top-level when. - # changeset: ## when PR contains any of those entries in the changeset - # - "^x-pack/metricbeat/module/aws/.*" - # - "^x-pack/metricbeat/Jenkinsfile.yml" - # - "^x-pack/libbeat/common/aws/.*" - # stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -62,78 +25,4 @@ stages: parameters: - "macosTest" tags: true ## for all the tags - stage: extended - # Ignore as long as there are test failures, see https://github.com/elastic/beats/issues/33036 -# macosM1: -# mage: "mage build unitTest" -# platforms: ## override default label in this specific stage. -# - "orka && darwin && aarch64" -# when: ## Override the top-level when. -# comments: -# - "/test metricbeat for macos-m1" -# labels: -# - "macos-m1" -# parameters: -# - "macosM1Test" -# tags: false ## for all the tags -# stage: extended - windows-2022: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2022" stage: mandatory - windows-2019: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2019" - stage: extended_win - windows-2016: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2016" - stage: mandatory - windows-2012: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2012-r2" - stage: extended_win - windows-11: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-11" - stage: extended_win - windows-10: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-10" - stage: extended_win - windows-8: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-8" - stage: extended_win - packaging-linux: - packaging-linux: "mage package" - e2e: - enabled: false - entrypoint: 'metricbeat-test.sh' - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/metricbeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack - packaging-arm: - packaging-arm: "mage package" - e2e: - enabled: false - platforms: ## override default label in this specific stage. - - "ubuntu-2204-aarch64" - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/metricbeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack From 20d2184c7036d9f3c4db85d5e3377d19e537e3ef Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 14:14:51 +0200 Subject: [PATCH 48/78] module test --- .buildkite/scripts/common.sh | 6 +++--- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d7773f2cb93..26af7b1d2ee 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -411,9 +411,9 @@ withModule() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE="${module_name}" + MODULE=${module_name} elif [ -d "${module_path}" ]; then - MODULE="" + MODULE='' fi echo "MODULE=$MODULE" } @@ -446,7 +446,7 @@ startCloudTestEnv() { withAWS() { export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY - export TEST_TAGS="${TEST_TAGS},aws" + export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" } if ! are_changed_only_paths "${docs_changeset[@]}" ; then diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c594075bfbb..e5f44636bd1 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -41,7 +41,7 @@ steps: # key: "mandatory-python-int-test" # command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" # env: - # MODULE: "$MODULE" + # MODULE: $MODULE # agents: # provider: "gcp" # image: "${IMAGE_UBUNTU_X86_64}" @@ -141,7 +141,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: "$MODULE" + MODULE: $MODULE agents: provider: "gcp" image: "family/core-ubuntu-2204" From 5fba5219f73fe20c8032d5a2ac7fbc9c281105b7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 16:18:31 +0200 Subject: [PATCH 49/78] debug --- .buildkite/scripts/install_tools.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index aa740c5b7e7..a7a2787e4f4 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -36,10 +36,6 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - startCloudTestEnv "${MODULE_DIR}" -fi - with_go "${GO_VERSION}" with_mage with_python @@ -47,6 +43,10 @@ with_dependencies config_git mage dumpVariables +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + startCloudTestEnv "${MODULE_DIR}" +fi + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 From f5a654544b51c6fe4876276fad90cdb5c9da8bb9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 20:12:58 +0200 Subject: [PATCH 50/78] debug --- .../generate_xpack_metricbeat_pipeline.sh | 8 +++---- .buildkite/scripts/setenv.sh | 4 ++++ dev-tools/mage/gotest.go | 23 ++++++++++--------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index e5f44636bd1..c56f782e4c7 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: ".buildkite/scripts/py_int_tests.sh" env: MODULE: "$MODULE" agents: @@ -145,9 +145,9 @@ if are_conditions_met_aws_tests; then agents: provider: "gcp" image: "family/core-ubuntu-2204" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dab4fd58e03..60ff9c578fe 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,6 +2,8 @@ set -euo pipefail REPO="beats" +TMP_FOLDER="tmp.${REPO}" +DOCKER_REGISTRY="docker.elastic.co" SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" @@ -24,6 +26,8 @@ export ASDF_MAGE_VERSION export PACKAGING_PLATFORMS export PACKAGING_ARM_PLATFORMS export REPO +export TMP_FOLDER +export DOCKER_REGISTRY export ASDF_TERRAFORM_VERSION export AWS_REGION diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index bc49c3e643c..d5793e60558 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -167,33 +167,34 @@ func DefaultTestBinaryArgs() TestBinaryArgs { // Use RACE_DETECTOR=true to enable the race detector. // Use MODULE=module to run only tests for `module`. func GoTestIntegrationForModule(ctx context.Context) error { + modulesFileInfo := []string{"aws", "gcp", "oracle", "sql"} module := EnvOr("MODULE", "") - modulesFileInfo, err := os.ReadDir("./module") - if err != nil { - return err - } + // modulesFileInfo, err := os.ReadDir("./module") + // if err != nil { + // return err + // } foundModule := false failedModules := make([]string, 0, len(modulesFileInfo)) for _, fi := range modulesFileInfo { // skip the ones that are not directories or with suffix @tmp, which are created by Jenkins build job - if !fi.IsDir() || strings.HasSuffix(fi.Name(), "@tmp") { + if !fi.IsDir() || strings.HasSuffix(fi, "@tmp") { continue } - if module != "" && module != fi.Name() { + if module != "" && module != fi { continue } foundModule = true // Set MODULE because only want that modules tests to run inside the testing environment. - env := map[string]string{"MODULE": fi.Name()} + env := map[string]string{"MODULE": fi} passThroughEnvs(env, IntegrationTestEnvVars()...) - runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env) + runners, err := NewIntegrationRunners(path.Join("./module", fi), env) if err != nil { - return fmt.Errorf("test setup failed for module %s: %w", fi.Name(), err) + return fmt.Errorf("test setup failed for module %s: %w", fi, err) } err = runners.Test("goIntegTest", func() error { - err := GoTest(ctx, GoTestIntegrationArgsForModule(fi.Name())) + err := GoTest(ctx, GoTestIntegrationArgsForModule(fi)) if err != nil { return err } @@ -201,7 +202,7 @@ func GoTestIntegrationForModule(ctx context.Context) error { }) if err != nil { // err will already be report to stdout, collect failed module to report at end - failedModules = append(failedModules, fi.Name()) + failedModules = append(failedModules, fi) } } if module != "" && !foundModule { From 5e71d65ff8bef2a19febeb78c26faf6687425aab Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 20:20:26 +0200 Subject: [PATCH 51/78] debug --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c56f782e4c7..2d3f690411d 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" env: MODULE: "$MODULE" agents: From 2aa399381d1ed6d2242f6ec51a6dc284798355fc Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 20:29:14 +0200 Subject: [PATCH 52/78] debug --- dev-tools/mage/gotest.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index d5793e60558..c041bafca5a 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -178,9 +178,9 @@ func GoTestIntegrationForModule(ctx context.Context) error { failedModules := make([]string, 0, len(modulesFileInfo)) for _, fi := range modulesFileInfo { // skip the ones that are not directories or with suffix @tmp, which are created by Jenkins build job - if !fi.IsDir() || strings.HasSuffix(fi, "@tmp") { - continue - } + // if !fi.IsDir() || strings.HasSuffix(fi, "@tmp") { + // continue + // } if module != "" && module != fi { continue } From 7545802b7fb2d0acf61307705fbe3bd296184bc9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 21:03:43 +0200 Subject: [PATCH 53/78] rolback changes --- .../generate_xpack_metricbeat_pipeline.sh | 4 +-- dev-tools/mage/gotest.go | 27 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 2d3f690411d..4cacf6791bd 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -28,7 +28,7 @@ steps: key: "mandatory-int-test" command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" env: - MODULE: "$MODULE" + MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -141,7 +141,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: $MODULE + MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index c041bafca5a..bc49c3e643c 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -167,34 +167,33 @@ func DefaultTestBinaryArgs() TestBinaryArgs { // Use RACE_DETECTOR=true to enable the race detector. // Use MODULE=module to run only tests for `module`. func GoTestIntegrationForModule(ctx context.Context) error { - modulesFileInfo := []string{"aws", "gcp", "oracle", "sql"} module := EnvOr("MODULE", "") - // modulesFileInfo, err := os.ReadDir("./module") - // if err != nil { - // return err - // } + modulesFileInfo, err := os.ReadDir("./module") + if err != nil { + return err + } foundModule := false failedModules := make([]string, 0, len(modulesFileInfo)) for _, fi := range modulesFileInfo { // skip the ones that are not directories or with suffix @tmp, which are created by Jenkins build job - // if !fi.IsDir() || strings.HasSuffix(fi, "@tmp") { - // continue - // } - if module != "" && module != fi { + if !fi.IsDir() || strings.HasSuffix(fi.Name(), "@tmp") { + continue + } + if module != "" && module != fi.Name() { continue } foundModule = true // Set MODULE because only want that modules tests to run inside the testing environment. - env := map[string]string{"MODULE": fi} + env := map[string]string{"MODULE": fi.Name()} passThroughEnvs(env, IntegrationTestEnvVars()...) - runners, err := NewIntegrationRunners(path.Join("./module", fi), env) + runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env) if err != nil { - return fmt.Errorf("test setup failed for module %s: %w", fi, err) + return fmt.Errorf("test setup failed for module %s: %w", fi.Name(), err) } err = runners.Test("goIntegTest", func() error { - err := GoTest(ctx, GoTestIntegrationArgsForModule(fi)) + err := GoTest(ctx, GoTestIntegrationArgsForModule(fi.Name())) if err != nil { return err } @@ -202,7 +201,7 @@ func GoTestIntegrationForModule(ctx context.Context) error { }) if err != nil { // err will already be report to stdout, collect failed module to report at end - failedModules = append(failedModules, fi) + failedModules = append(failedModules, fi.Name()) } } if module != "" && !foundModule { From 929553f48b70fe4663105441f6e65ada7bff07ad Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 21:04:35 +0200 Subject: [PATCH 54/78] rolback changes --- Jenkinsfile.yml | 18 ++++- x-pack/metricbeat/Jenkinsfile.yml | 111 ++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 3e2431a7eed..811b4ae985b 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -1,6 +1,22 @@ projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "heartbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/libbeat" - "x-pack/metricbeat" - + - "x-pack/osquerybeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" ## Changeset macros that are defined here and used in each specific 2.0 pipeline. changeset: diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 6d0d2319db7..27574b62ada 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,6 +13,43 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-22" ## default label for all the stages stages: + unitTest: + mage: "mage build unitTest" + stage: mandatory + goIntegTest: + mage: "mage goIntegTest" + withModule: true + stage: mandatory + pythonIntegTest: + mage: "mage pythonIntegTest" + withModule: true + stage: mandatory + cloud: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## run the cloud tests for the given modules. + - "x-pack/metricbeat/module/aws" + when: ## Override the top-level when. + parameters: + - "awsCloudTests" + comments: + - "/test x-pack/metricbeat for aws cloud" + labels: + - "aws" + # stage: extended + # Skip test until fixed https://github.com/elastic/beats/issues/36425 + #cloudAWS: + # cloud: "mage build test goIntegTest" + # withAWS: true ## Enable the tests to run in AWS + # withModule: true ## run the ITs only if the changeset affects a specific module. + # dirs: ## run the cloud tests for the given modules. + # - "x-pack/metricbeat/module/aws" + # when: ## Override the top-level when. + # changeset: ## when PR contains any of those entries in the changeset + # - "^x-pack/metricbeat/module/aws/.*" + # - "^x-pack/metricbeat/Jenkinsfile.yml" + # - "^x-pack/libbeat/common/aws/.*" + # stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -25,4 +62,78 @@ stages: parameters: - "macosTest" tags: true ## for all the tags + stage: extended + # Ignore as long as there are test failures, see https://github.com/elastic/beats/issues/33036 +# macosM1: +# mage: "mage build unitTest" +# platforms: ## override default label in this specific stage. +# - "orka && darwin && aarch64" +# when: ## Override the top-level when. +# comments: +# - "/test metricbeat for macos-m1" +# labels: +# - "macos-m1" +# parameters: +# - "macosM1Test" +# tags: false ## for all the tags +# stage: extended + windows-2022: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2022" stage: mandatory + windows-2019: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" + stage: extended_win + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + stage: mandatory + windows-2012: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2012-r2" + stage: extended_win + windows-11: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-11" + stage: extended_win + windows-10: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-10" + stage: extended_win + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + stage: extended_win + packaging-linux: + packaging-linux: "mage package" + e2e: + enabled: false + entrypoint: 'metricbeat-test.sh' + stage: packaging + when: + branches: false ## Only on a PR basis for the time being + tags: false ## packaging on branches/tags is already in place with the downstream build. + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/.*" + - "@xpack" ## special token regarding the changeset for the xpack + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "ubuntu-2204-aarch64" + stage: packaging + when: + branches: false ## Only on a PR basis for the time being + tags: false ## packaging on branches/tags is already in place with the downstream build. + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/.*" + - "@xpack" ## special token regarding the changeset for the xpack From 9303a2a62884910ab596ecdd5cb1f46a36f5e45e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 6 Mar 2024 21:31:59 +0200 Subject: [PATCH 55/78] rolback changes --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 4cacf6791bd..bb001d1b923 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,8 +27,8 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - env: - MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved + # env: + # MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -140,8 +140,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" - env: - MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved + # env: + # MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" From 700f8c131816e909dc7e163176650ca7a5a9c0f9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 08:22:29 +0200 Subject: [PATCH 56/78] test --- .buildkite/scripts/common.sh | 12 ++--- .../generate_xpack_metricbeat_pipeline.sh | 44 ++++++++++--------- x-pack/metricbeat/module/aws/variables.tf | 4 ++ 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 26af7b1d2ee..8902fc8c7e2 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -426,6 +426,12 @@ terraformInit() { popd > /dev/null } +withAWS() { + export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY + export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY + export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" +} + startCloudTestEnv() { local dir=$1 withAWS @@ -443,12 +449,6 @@ startCloudTestEnv() { popd > /dev/null } -withAWS() { - export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY - export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -} - if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bb001d1b923..d8851fb9a81 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -26,27 +26,29 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - # env: - # MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved + command: ".buildkite/scripts/go_int_tests.sh" + env: + MODULE: $MODULE #TODO: uncomment when the issue is solved agents: provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" + image: "family/core-ubuntu-2204" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - # - label: ":python: Python Integration Tests" - # key: "mandatory-python-int-test" - # command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - # env: - # MODULE: $MODULE - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "family/core-ubuntu-2204" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" # - label: ":windows: Windows Unit Tests - {{matrix.image}}" # command: ".buildkite/scripts/win_unit_tests.ps1" @@ -141,13 +143,13 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" # env: - # MODULE: "aws" #TODO: replace with "$MODULE" when the issue is solved + # MODULE: $MODULE #TODO: uncomment when the issue is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index e4b95d471b2..cceae9e9504 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,3 +20,7 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } + +variable "TESTVAR" { + default = "unknown-variable" +} From d649de32fc536b1da2e5865e26d03f061ffebbcb Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 10:32:03 +0200 Subject: [PATCH 57/78] test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index d8851fb9a81..7a990c09921 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -142,8 +142,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" - # env: - # MODULE: $MODULE #TODO: uncomment when the issue is solved + env: + MODULE: $MODULE #TODO: uncomment when the issue is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" From 56d24bf3ec5591b6b016fc99c6814c1c5a0a17c1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 10:53:49 +0200 Subject: [PATCH 58/78] remove aws changes --- x-pack/metricbeat/module/aws/variables.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index cceae9e9504..e4b95d471b2 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,7 +20,3 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } - -variable "TESTVAR" { - default = "unknown-variable" -} From ad0492aa429febbf38e9096730fb4a37294f0db8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 11:32:06 +0200 Subject: [PATCH 59/78] test with hardcoded MODULE=aws --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 7a990c09921..f20798274c0 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -28,7 +28,7 @@ steps: key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" env: - MODULE: $MODULE #TODO: uncomment when the issue is solved + MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -41,7 +41,7 @@ steps: key: "mandatory-python-int-test" command: ".buildkite/scripts/py_int_tests.sh" env: - MODULE: $MODULE + MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -143,7 +143,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: $MODULE #TODO: uncomment when the issue is solved + MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved agents: provider: "gcp" image: "family/core-ubuntu-2204" From d532855ad3028bf42c859dc0fcd5a4c8c27362f5 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 12:20:38 +0200 Subject: [PATCH 60/78] test with hardcoded MODULE=aws --- .buildkite/scripts/cloud_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 33b2ac797d9..5d43cf52ad2 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -7,7 +7,7 @@ set -euo pipefail echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null -mage build test +MODULE=aws mage build test popd > /dev/null From 5823883848013875d4e31e1fd91381e320354fb3 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 12:59:10 +0200 Subject: [PATCH 61/78] test with hardcoded MODULE='' --- .buildkite/scripts/cloud_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 5d43cf52ad2..999c3aed810 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -7,7 +7,7 @@ set -euo pipefail echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null -MODULE=aws mage build test +MODULE='' mage build test popd > /dev/null From 6b3e9de2cce5736746f3c27c9889c572b62d982e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 14:30:06 +0200 Subject: [PATCH 62/78] test --- .buildkite/scripts/common.sh | 5 +++-- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 8902fc8c7e2..8075fe7d0d3 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -411,9 +411,10 @@ withModule() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE=${module_name} + export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue with the loop (cycle) is solved + # export MODULE=${module_name} elif [ -d "${module_path}" ]; then - MODULE='' + export MODULE='' fi echo "MODULE=$MODULE" } diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index f20798274c0..c90020e3c18 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -28,7 +28,7 @@ steps: key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" env: - MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved + MODULE: $MODULE agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -41,7 +41,7 @@ steps: key: "mandatory-python-int-test" command: ".buildkite/scripts/py_int_tests.sh" env: - MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved + MODULE: $MODULE agents: provider: "gcp" image: "family/core-ubuntu-2204" @@ -143,7 +143,7 @@ if are_conditions_met_aws_tests; then key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" env: - MODULE: "aws" #TODO: it has to be replaced with "$MODULE" when the issue with the loop (cycle) is solved + MODULE: $MODULE agents: provider: "gcp" image: "family/core-ubuntu-2204" From bd864ec7437820d5613e1324611c0827a211f2e1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 15:07:29 +0200 Subject: [PATCH 63/78] test --- .buildkite/scripts/common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 8075fe7d0d3..7c9ce97bff7 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -411,10 +411,10 @@ withModule() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue with the loop (cycle) is solved - # export MODULE=${module_name} + export MODULE=${module_name} elif [ -d "${module_path}" ]; then - export MODULE='' + export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue with the loop (cycle) is solved + # export MODULE='' fi echo "MODULE=$MODULE" } From 6ff685b218e324d349cd7fd5a731cf929eb37a11 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 15:11:48 +0200 Subject: [PATCH 64/78] test --- .buildkite/scripts/cloud_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 999c3aed810..33b2ac797d9 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -7,7 +7,7 @@ set -euo pipefail echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null -MODULE='' mage build test +mage build test popd > /dev/null From 99075798d45e8e89bb20d43b57d2d399e2866ac9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 16:11:32 +0200 Subject: [PATCH 65/78] full test --- .buildkite/scripts/common.sh | 4 +- .../generate_xpack_metricbeat_pipeline.sh | 214 +++++++++--------- 2 files changed, 108 insertions(+), 110 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7c9ce97bff7..47809018f20 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -413,8 +413,8 @@ withModule() { if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then export MODULE=${module_name} elif [ -d "${module_path}" ]; then - export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue with the loop (cycle) is solved - # export MODULE='' + export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + # export MODULE='' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi echo "MODULE=$MODULE" } diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c90020e3c18..a942c5fe410 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -15,14 +15,14 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - # - label: ":linux: Ubuntu Unit Tests" - # key: "mandatory-linux-unit-test" - # command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -50,58 +50,58 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # key: "mandatory-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_2016}" - # - "${IMAGE_WIN_2022}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - # - group: "Extended Windows Tests" - # key: "extended-win-tests" - # steps: - # - label: ":windows: Windows 10 Unit Tests" - # key: "extended-win-10-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Windows 11 Unit Tests" - # key: "extended-win-11-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_11}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Win 2019 Unit Tests" - # key: "extended-win-2019-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2019}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -121,21 +121,19 @@ if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then YAML fi -# if are_conditions_met_macos_tests; then -# cat >> $pipelineName <<- YAML - -# - label: ":mac: MacOS Unit Tests" -# key: "extended-macos-unit-tests" -# command: ".buildkite/scripts/unit_tests.sh" -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_X86_64}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML -# YAML -# fi + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -#TODO: uncomment the commented-out below step when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved +YAML +fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML @@ -155,41 +153,41 @@ if are_conditions_met_aws_tests; then YAML fi -# echo "Check and add the Packaging into the pipeline" -# if are_conditions_met_packaging; then -# cat >> $pipelineName <<- YAML - -# - wait: ~ -# depends_on: -# - step: "mandatory-tests" -# allow_failure: false - -# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 -# key: "packaging" -# steps: -# - label: ":linux: Packaging Linux" -# key: "packaging-linux" -# command: "cd $BEATS_PROJECT_NAME && mage package" -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# env: -# PLATFORMS: "${PACKAGING_PLATFORMS}" - -# - label: ":linux: Packaging ARM" -# key: "packaging-arm" -# command: "cd $BEATS_PROJECT_NAME && mage package" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "${AWS_ARM_INSTANCE_TYPE}" -# env: -# PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" -# PACKAGES: "docker" - -# YAML -# fi +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public cat $pipelineName From 48e1c0891177938701d08d015bc0f32440afb2b8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 7 Mar 2024 17:18:46 +0200 Subject: [PATCH 66/78] change images and add cleanup --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/cloud_tests.sh | 8 ++------ .buildkite/scripts/common.sh | 16 ++++++++++++++++ .../generate_xpack_metricbeat_pipeline.sh | 12 ++++++------ .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 1 + 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c20a55a4250..282fb5a0085 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -41,7 +41,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_SECRET_KEY BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 33b2ac797d9..1fa693ef38b 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -4,15 +4,11 @@ source .buildkite/scripts/install_tools.sh set -euo pipefail +trap 'cleanup; unset_secrets' EXIT + echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null mage build test popd > /dev/null - -echo "---Terraform Cleanup" -.ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK - -echo "---Docker Compose Cleanup" -docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 47809018f20..b58bec7c2eb 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -450,6 +450,22 @@ startCloudTestEnv() { popd > /dev/null } +cleanup() { + echo "---Terraform Cleanup" + .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + + echo "---Docker Compose Cleanup" + docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then + unset "$var" + fi + done +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index a942c5fe410..9fec4448504 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -31,8 +31,8 @@ steps: MODULE: $MODULE agents: provider: "gcp" - image: "family/core-ubuntu-2204" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" @@ -44,8 +44,8 @@ steps: MODULE: $MODULE agents: provider: "gcp" - image: "family/core-ubuntu-2204" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" @@ -144,8 +144,8 @@ if are_conditions_met_aws_tests; then MODULE: $MODULE agents: provider: "gcp" - image: "family/core-ubuntu-2204" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 83a7fe886c8..81756564919 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -3,6 +3,7 @@ name: "beats-xpack-metricbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/general-windows-10" IMAGE_WIN_11: "family/general-windows-11" From fd09867e41f75354b00c4b89f2aabd9e0b7f76a1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 09:15:18 +0200 Subject: [PATCH 67/78] work with remarks --- .buildkite/scripts/common.sh | 15 +++++++++------ .buildkite/scripts/setenv.sh | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b58bec7c2eb..981c2001658 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -399,7 +399,9 @@ config_git() { fi } -withModule() { +getModuleNameFromTheChangeSet() { + # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. + # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. local module_path=$1 if [[ "$module_path" == *"x-pack/"* ]]; then local pattern=("$XPACK_MODULE_PATTERN") @@ -411,12 +413,11 @@ withModule() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - export MODULE=${module_name} + return "${module_name}" elif [ -d "${module_path}" ]; then - export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # export MODULE='' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + return "aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + # return '' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi - echo "MODULE=$MODULE" } terraformInit() { @@ -428,6 +429,7 @@ terraformInit() { } withAWS() { + # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names. export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" @@ -482,7 +484,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "${MODULE_DIR}" + MODULE=$(getModuleNameFromTheChangeSet "${MODULE_DIR}") + export MODULE fi check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 60ff9c578fe..cba16261a31 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -61,9 +61,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" - export TEST_TAGS="oracle" + export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + # run the cloud tests for the given modules. export MODULE_DIR="x-pack/metricbeat/module/aws" fi From 0db069634c866ad3aeb4450651495e1ed2ae9042 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 09:51:58 +0200 Subject: [PATCH 68/78] replace return with echo --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 981c2001658..6227fe11d1c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -413,9 +413,9 @@ getModuleNameFromTheChangeSet() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - return "${module_name}" + echo "${module_name}" elif [ -d "${module_path}" ]; then - return "aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + echo "aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved # return '' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi } From 6c5968099b655885527a2ddbcc756a74cb02c154 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 10:01:30 +0200 Subject: [PATCH 69/78] revert exporting var --- .buildkite/scripts/common.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6227fe11d1c..dadec9703da 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -413,10 +413,10 @@ getModuleNameFromTheChangeSet() { local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - echo "${module_name}" + export MODULE=${module_name} elif [ -d "${module_path}" ]; then - echo "aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # return '' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + # export MODULE='' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi } @@ -484,8 +484,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - MODULE=$(getModuleNameFromTheChangeSet "${MODULE_DIR}") - export MODULE + getModuleNameFromTheChangeSet "${MODULE_DIR}" fi check_and_set_beat_vars From 1ca09df329629f66be3f9a67863103b5ffe453ff Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 10:07:32 +0200 Subject: [PATCH 70/78] update method name and description --- .buildkite/scripts/common.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index dadec9703da..ec2908e997b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -399,9 +399,10 @@ config_git() { fi } -getModuleNameFromTheChangeSet() { +defineModuleFromTheChangeSet() { # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. + # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module local module_path=$1 if [[ "$module_path" == *"x-pack/"* ]]; then local pattern=("$XPACK_MODULE_PATTERN") @@ -484,7 +485,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - getModuleNameFromTheChangeSet "${MODULE_DIR}" + defineModuleFromTheChangeSet "${MODULE_DIR}" fi check_and_set_beat_vars From b1fdc2ceb7011f35157b6dea39858e5573f73166 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 11:28:00 +0200 Subject: [PATCH 71/78] change cloud_tests --- .buildkite/scripts/cloud_tests.sh | 12 ++++++++++++ .buildkite/scripts/common.sh | 4 ---- .buildkite/scripts/install_tools.sh | 4 ---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 1fa693ef38b..1302bca0b4f 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -9,6 +9,18 @@ trap 'cleanup; unset_secrets' EXIT echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null +# Cloud stage for x-pack/metricbeat +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${BUILDKITE_STEP_KEY}" == "mandatory-int-test" || "${BUILDKITE_STEP_KEY}" == "mandatory-python-int-test" || "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + # Configure MODULE env variable. + defineModuleFromTheChangeSet "${MODULE_DIR}" + fi + if [[ "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + # Start the required services for the required + startCloudTestEnv "${MODULE_DIR}" + fi +fi + mage build test popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ec2908e997b..4d702f58615 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -484,8 +484,4 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - defineModuleFromTheChangeSet "${MODULE_DIR}" -fi - check_and_set_beat_vars diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index a7a2787e4f4..80e70ae96c5 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -43,10 +43,6 @@ with_dependencies config_git mage dumpVariables -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - startCloudTestEnv "${MODULE_DIR}" -fi - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 From 25547b4ef8c75e9c356573b39e0bc25c0bf5e022 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 11:56:39 +0200 Subject: [PATCH 72/78] move conditions to the pre-command hook --- .buildkite/hooks/pre-command | 41 +++++++++++++------------------ .buildkite/scripts/cloud_tests.sh | 12 --------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 282fb5a0085..67580d63901 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,26 +2,9 @@ set -euo pipefail -AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" +source .buildkite/scripts/common.sh -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} +AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh @@ -41,9 +24,19 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then - BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_SECRET_KEY - BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_ACCESS_KEY +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${BUILDKITE_STEP_KEY}" == "mandatory-int-test" || "${BUILDKITE_STEP_KEY}" == "mandatory-python-int-test" || "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + # Configure MODULE env variable. + defineModuleFromTheChangeSet "${MODULE_DIR}" + fi + if [[ "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + # Start the required services for the required + startCloudTestEnv "${MODULE_DIR}" + + # export AWS masked credentials + BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_SECRET_KEY + BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_ACCESS_KEY + fi fi diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 1302bca0b4f..1fa693ef38b 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -9,18 +9,6 @@ trap 'cleanup; unset_secrets' EXIT echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null -# Cloud stage for x-pack/metricbeat -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${BUILDKITE_STEP_KEY}" == "mandatory-int-test" || "${BUILDKITE_STEP_KEY}" == "mandatory-python-int-test" || "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - # Configure MODULE env variable. - defineModuleFromTheChangeSet "${MODULE_DIR}" - fi - if [[ "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - # Start the required services for the required - startCloudTestEnv "${MODULE_DIR}" - fi -fi - mage build test popd > /dev/null From c96b847d7170701b89840b4b497985b438166273 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 11 Mar 2024 12:00:35 +0200 Subject: [PATCH 73/78] revert all last changes --- .buildkite/hooks/pre-command | 41 +++++++++++++++++------------ .buildkite/scripts/common.sh | 4 +++ .buildkite/scripts/install_tools.sh | 4 +++ 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 67580d63901..282fb5a0085 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,10 +2,27 @@ set -euo pipefail -source .buildkite/scripts/common.sh - AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -24,19 +41,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${BUILDKITE_STEP_KEY}" == "mandatory-int-test" || "${BUILDKITE_STEP_KEY}" == "mandatory-python-int-test" || "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - # Configure MODULE env variable. - defineModuleFromTheChangeSet "${MODULE_DIR}" - fi - if [[ "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - # Start the required services for the required - startCloudTestEnv "${MODULE_DIR}" - - # export AWS masked credentials - BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_SECRET_KEY - BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_ACCESS_KEY - fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then + BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_SECRET_KEY + BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_ACCESS_KEY fi diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 4d702f58615..ec2908e997b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -484,4 +484,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + defineModuleFromTheChangeSet "${MODULE_DIR}" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 80e70ae96c5..a7a2787e4f4 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -43,6 +43,10 @@ with_dependencies config_git mage dumpVariables +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then + startCloudTestEnv "${MODULE_DIR}" +fi + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 From 1057a948a9dfde823529f3fc6297c96795164c5c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 12:00:39 +0200 Subject: [PATCH 74/78] test Victor's suggestion --- .buildkite/scripts/cloud_tests.sh | 12 +++++++++++- .buildkite/scripts/common.sh | 7 ++----- .../scripts/generate_xpack_metricbeat_pipeline.sh | 6 ------ .buildkite/scripts/install_tools.sh | 4 ---- .buildkite/scripts/setenv.sh | 5 ----- 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 1fa693ef38b..47747aa570c 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -1,11 +1,21 @@ #!/usr/bin/env bash +# What Terraform Module will run +export MODULE_DIR="x-pack/metricbeat/module/aws" + source .buildkite/scripts/install_tools.sh set -euo pipefail -trap 'cleanup; unset_secrets' EXIT +trap 'teardown; unset_secrets' EXIT + +# Set the MODULE env variable if possible +defineModuleFromTheChangeSet "${MODULE_DIR}" + +# Prepare the cloud resources using Terraform +startCloudTestEnv "${MODULE_DIR}" +# Run tests echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" pushd "${BEATS_PROJECT_NAME}" > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ec2908e997b..6ae26ee5479 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -453,7 +453,8 @@ startCloudTestEnv() { popd > /dev/null } -cleanup() { +teardown() { + # Teardown resources after using them echo "---Terraform Cleanup" .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK @@ -484,8 +485,4 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - defineModuleFromTheChangeSet "${MODULE_DIR}" -fi - check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 9fec4448504..32ca5afd8f2 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,8 +27,6 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" - env: - MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -40,8 +38,6 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: ".buildkite/scripts/py_int_tests.sh" - env: - MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -140,8 +136,6 @@ if are_conditions_met_aws_tests; then - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" - env: - MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index a7a2787e4f4..80e70ae96c5 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -43,10 +43,6 @@ with_dependencies config_git mage dumpVariables -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "${BUILDKITE_STEP_KEY}" == "extended-cloud-test" ]]; then - startCloudTestEnv "${MODULE_DIR}" -fi - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index cba16261a31..974886061d4 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -63,8 +63,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export DOCKER_PULL="0" export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - # run the cloud tests for the given modules. - export MODULE_DIR="x-pack/metricbeat/module/aws" -fi From 84e8305daa6c7b268b5e25e5f76bfabc87d86213 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 12:05:48 +0200 Subject: [PATCH 75/78] test Victor's suggestion --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 32ca5afd8f2..c9c65a5e757 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -133,7 +133,7 @@ fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ${MODULE^^} Cloud Tests" + - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" agents: From 141c990fb8b80def88fed8bfbbe242bb456d4f1a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 14:18:24 +0200 Subject: [PATCH 76/78] change defineModuleFromTheChangeSet --- .buildkite/scripts/cloud_tests.sh | 2 +- .buildkite/scripts/common.sh | 14 +++++++------- x-pack/metricbeat/module/aws/usage/manifest.yml | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 47747aa570c..12d10dc3bbb 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -10,7 +10,7 @@ set -euo pipefail trap 'teardown; unset_secrets' EXIT # Set the MODULE env variable if possible -defineModuleFromTheChangeSet "${MODULE_DIR}" +defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" # Prepare the cloud resources using Terraform startCloudTestEnv "${MODULE_DIR}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6ae26ee5479..548131ae3e4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -403,19 +403,19 @@ defineModuleFromTheChangeSet() { # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module - local module_path=$1 - if [[ "$module_path" == *"x-pack/"* ]]; then + local project_path=$1 + if [[ "$project_path" == *"x-pack/"* ]]; then local pattern=("$XPACK_MODULE_PATTERN") else local pattern=("$OSS_MODULE_PATTERN") fi - local module_name="${module_path#*module/}" - local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') - local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" - local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + local module_name="${project_path#*module/}" + local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') + local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" + local exclude=("^(${project_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then export MODULE=${module_name} - elif [ -d "${module_path}" ]; then + elif [ -d "${project_path}" ]; then export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved # export MODULE='' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi diff --git a/x-pack/metricbeat/module/aws/usage/manifest.yml b/x-pack/metricbeat/module/aws/usage/manifest.yml index 4ef64eee065..b47f43c2277 100644 --- a/x-pack/metricbeat/module/aws/usage/manifest.yml +++ b/x-pack/metricbeat/module/aws/usage/manifest.yml @@ -6,3 +6,5 @@ input: metrics: - namespace: AWS/Usage statistic: ["Sum"] + - namespace: AWS/Usage + statistic: ["Average"] From 3cf8aa0bc367dbb61cf4693227ab0325cf77babf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 15:11:30 +0200 Subject: [PATCH 77/78] change defineModuleFromTheChangeSet --- .buildkite/scripts/common.sh | 29 +++++++++++++------ .../metricbeat/module/aws/usage/manifest.yml | 2 -- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 548131ae3e4..ebb15c937dd 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -404,20 +404,31 @@ defineModuleFromTheChangeSet() { # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module local project_path=$1 + local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') + local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" + local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if [[ "$project_path" == *"x-pack/"* ]]; then local pattern=("$XPACK_MODULE_PATTERN") else local pattern=("$OSS_MODULE_PATTERN") fi - local module_name="${project_path#*module/}" - local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') - local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" - local exclude=("^(${project_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - export MODULE=${module_name} - elif [ -d "${project_path}" ]; then - export MODULE="aws" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # export MODULE='' # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + local changed_modules="" + local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d) + for module_dir in $module_dirs; do + if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then + if [[ -z "$changed_modules" ]]; then + changed_modules=$(basename "$module_dir") + else + changed_modules+=",$(basename "$module_dir")" + fi + fi + done + if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + export MODULE="aws" + else + export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved fi } diff --git a/x-pack/metricbeat/module/aws/usage/manifest.yml b/x-pack/metricbeat/module/aws/usage/manifest.yml index b47f43c2277..4ef64eee065 100644 --- a/x-pack/metricbeat/module/aws/usage/manifest.yml +++ b/x-pack/metricbeat/module/aws/usage/manifest.yml @@ -6,5 +6,3 @@ input: metrics: - namespace: AWS/Usage statistic: ["Sum"] - - namespace: AWS/Usage - statistic: ["Average"] From f4328229bf04ba42f9d5743a0270bcf5fa0c8546 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 15:33:12 +0200 Subject: [PATCH 78/78] add true for teardown --- .buildkite/scripts/cloud_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 12d10dc3bbb..9e7ce95be66 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -7,7 +7,7 @@ source .buildkite/scripts/install_tools.sh set -euo pipefail -trap 'teardown; unset_secrets' EXIT +trap 'teardown || true; unset_secrets' EXIT # Set the MODULE env variable if possible defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"