From dd8b27e41107d5854d5c1eeb33aa94da14c71fdf Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:42:06 +0200 Subject: [PATCH] migrate xpack-metricbeat (#38081) * migrate xpack-metricbeat (cherry picked from commit 843010c1cbddd201bb4c08a79fd649a0af3c04ce) # Conflicts: # .buildkite/scripts/generate_packetbeat_pipeline.sh --- .buildkite/hooks/pre-command | 30 ++- .buildkite/scripts/cloud_tests.sh | 24 +++ .buildkite/scripts/common.sh | 171 ++++++++++++++-- .../scripts/generate_metricbeat_pipeline.sh | 4 +- .../scripts/generate_packetbeat_pipeline.sh | 8 +- .../scripts/generate_winlogbeat_pipeline.sh | 2 +- .../generate_xpack_metricbeat_pipeline.sh | 190 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 21 +- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 8 +- .../x-pack/pipeline.xpack.metricbeat.yml | 60 +++++- 10 files changed, 490 insertions(+), 28 deletions(-) create mode 100755 .buildkite/scripts/cloud_tests.sh create mode 100755 .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0ac7c51099c..282fb5a0085 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,6 +2,27 @@ set -euo pipefail +AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -12,10 +33,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" exit 0 fi fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then + BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_SECRET_KEY + BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_ACCESS_KEY +fi diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh new file mode 100755 index 00000000000..9e7ce95be66 --- /dev/null +++ b/.buildkite/scripts/cloud_tests.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# What Terraform Module will run +export MODULE_DIR="x-pack/metricbeat/module/aws" + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +trap 'teardown || true; unset_secrets' EXIT + +# Set the MODULE env variable if possible +defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" + +# Prepare the cloud resources using Terraform +startCloudTestEnv "${MODULE_DIR}" + +# Run tests +echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage build test + +popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b797ec718aa..ebb15c937dd 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,6 +9,8 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -17,6 +19,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" +[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" +[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" +[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" metricbeat_changeset=( "^metricbeat/.*" @@ -81,6 +86,30 @@ packaging_changeset=( ".go-version" ) +case "${BUILDKITE_PIPELINE_SLUG}" in + "beats-metricbeat") + BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]} + ;; + "beats-libbeat") + BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} + ;; + "beats-packetbeat") + BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} + ;; + "beats-winlogbeat") + BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} + ;; + "beats-xpack-libbeat") + BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} + ;; + "beats-xpack-metricbeat") + BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} + ;; + *) + echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + ;; +esac + check_and_set_beat_vars() { if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - @@ -89,8 +118,8 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -101,8 +130,8 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -113,8 +142,10 @@ check_and_set_beat_vars() { BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" - BAETS_GH_MACOS_LABEL="macOS" - BAETS_GH_ARM_LABEL="arm" + BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" + BEATS_GH_MACOS_LABEL="macOS" + BEATS_GH_ARM_LABEL="arm" + BEATS_GH_AWS_LABEL="aws" } with_docker_compose() { @@ -127,6 +158,19 @@ with_docker_compose() { docker-compose version } +with_Terraform() { + echo "Setting up the Terraform environment..." + local path_to_file="${WORKSPACE}/terraform.zip" + create_workspace + check_platform_architeture + retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip" + unzip -q ${path_to_file} -d ${BIN}/ + rm ${path_to_file} + chmod +x ${BIN}/terraform + export PATH="${BIN}:${PATH}" + terraform version +} + create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -168,6 +212,8 @@ with_mage() { for pkg in "${install_packages[@]}"; do go install "${pkg}@latest" done + echo "Download modules to local cache" + retry 3 go mod download } with_go() { @@ -283,9 +329,15 @@ are_paths_changed() { are_changed_only_paths() { local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + local changed_files=($(git diff --name-only HEAD@{1} HEAD)) + local matched_files=() + for pattern in "${patterns[@]}"; do + local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) + if [ "${#matched[@]}" -gt 0 ]; then + matched_files+=("${matched[@]}") + fi + done + if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then return 0 fi return 1 @@ -301,7 +353,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -311,8 +363,19 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + fi + fi + return 1 +} + +are_conditions_met_aws_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -336,19 +399,101 @@ config_git() { fi } +defineModuleFromTheChangeSet() { + # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. + # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. + # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module + local project_path=$1 + local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') + local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" + local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + + if [[ "$project_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local changed_modules="" + local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d) + for module_dir in $module_dirs; do + if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then + if [[ -z "$changed_modules" ]]; then + changed_modules=$(basename "$module_dir") + else + changed_modules+=",$(basename "$module_dir")" + fi + fi + done + if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + export MODULE="aws" + else + export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved + fi +} + +terraformInit() { + local dir=$1 + echo "Terraform Init on $dir" + pushd "${dir}" > /dev/null + terraform init + popd > /dev/null +} + +withAWS() { + # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names. + export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY + export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY + export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" +} + +startCloudTestEnv() { + local dir=$1 + withAWS + echo "--- Run docker-compose services for emulated cloud env" + docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + with_Terraform + terraformInit "$dir" + export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') + export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" + export TF_VAR_CREATED_DATE=$(date +%s) + export TF_VAR_ENVIRONMENT="ci" + export TF_VAR_REPO="${REPO}" + pushd "${dir}" > /dev/null + terraform apply -auto-approve + popd > /dev/null +} + +teardown() { + # Teardown resources after using them + echo "---Terraform Cleanup" + .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + + echo "---Docker Compose Cleanup" + docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then + unset "$var" + fi + done +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then - ONLY_DOCS="false" + export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." else echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." fi if are_paths_changed "${go_mod_changeset[@]}" ; then - GO_MOD_CHANGES="true" + export GO_MOD_CHANGES="true" fi if are_paths_changed "${packaging_changeset[@]}" ; then - PACKAGING_CHANGES="true" + export PACKAGING_CHANGES="true" fi check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index e91896eb70c..477f8fb25a3 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -149,7 +149,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -159,7 +159,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 1186fd49c01..eb314248133 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -92,7 +92,7 @@ else exit 0 fi -if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -150,7 +150,11 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: +<<<<<<< HEAD PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" +======= + PLATFORMS: "${PACKAGING_PLATFORMS}" +>>>>>>> 843010c1cb (migrate xpack-metricbeat (#38081)) - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -160,7 +164,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index 1eb1b459c92..ce812016e79 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -96,7 +96,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" YAML diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh new file mode 100755 index 00000000000..c9c65a5e757 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-metricbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +#TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved +if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +# if [[ are_conditions_met_macos_tests ]]; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_aws_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Cloud Tests" + key: "extended-cloud-test" + command: ".buildkite/scripts/cloud_tests.sh" + agents: + provider: "gcp" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 25121de212f..974886061d4 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -1,13 +1,20 @@ #!/usr/bin/env bash set -euo pipefail - +REPO="beats" +TMP_FOLDER="tmp.${REPO}" +DOCKER_REGISTRY="docker.elastic.co" SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) +ASDF_MAGE_VERSION="1.15.0" +ASDF_TERRAFORM_VERSION="1.0.2" +PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +PACKAGING_ARM_PLATFORMS="linux/arm64" +AWS_REGION="eu-central-1" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -15,6 +22,14 @@ export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION +export ASDF_MAGE_VERSION +export PACKAGING_PLATFORMS +export PACKAGING_ARM_PLATFORMS +export REPO +export TMP_FOLDER +export DOCKER_REGISTRY +export ASDF_TERRAFORM_VERSION +export AWS_REGION exportVars() { local platform_type="$(uname)" @@ -41,10 +56,10 @@ exportVars() { fi } - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" + export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 01695fa4fb6..0c745b1a09d 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -20,7 +20,7 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - run_xpack_libbeat" + - select: "Libbeat - run_xpack_libbeat" key: "run_xpack_libbeat" options: - label: "True" @@ -28,7 +28,7 @@ steps: - label: "False" value: "false" default: "false" - - select: "Packetbeat - run_xpack_libbeat_arm_tests" + - select: "Libbeat - run_xpack_libbeat_arm_tests" key: "run_xpack_libbeat_arm_tests" options: - label: "True" @@ -42,8 +42,8 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic packetbeat pipeline" - key: "packetbeat-pipeline" + - label: ":linux: Load dynamic x-pack libbeat pipeline" + key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: - github_commit_status: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 34321b61161..81756564919 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -1,5 +1,61 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-metricbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Metricbeat - run_xpack_metricbeat" + key: "run_xpack_metricbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_macos_tests" + key: "run_xpack_metricbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_aws_tests" + key: "run_xpack_metricbeat_aws_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack metricbeat pipeline" + key: "metricbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"