Skip to content

Commit

Permalink
migrate xpack-metricbeat (#38081)
Browse files Browse the repository at this point in the history
* migrate xpack-metricbeat

(cherry picked from commit 843010c)
  • Loading branch information
sharbuz authored and mergify[bot] committed Mar 12, 2024
1 parent 000992d commit 64ba153
Show file tree
Hide file tree
Showing 10 changed files with 487 additions and 29 deletions.
30 changes: 29 additions & 1 deletion .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,27 @@

set -euo pipefail

AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"

retry() {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ $count -lt "$retries" ]; then
>&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
sleep $wait
else
>&2 echo "Retry $count/$retries exited $exit, no more retries left."
return $exit
fi
done
return 0
}

if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
source .buildkite/env-scripts/env.sh
source .buildkite/env-scripts/util.sh
Expand All @@ -12,10 +33,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
fi
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
source .buildkite/scripts/setenv.sh
if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
echo "Skipped pre-command when running the Upload pipeline"
exit 0
fi
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then
BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
export BEATS_AWS_SECRET_KEY
BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
export BEATS_AWS_ACCESS_KEY
fi
24 changes: 24 additions & 0 deletions .buildkite/scripts/cloud_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env bash

# What Terraform Module will run
export MODULE_DIR="x-pack/metricbeat/module/aws"

source .buildkite/scripts/install_tools.sh

set -euo pipefail

trap 'teardown || true; unset_secrets' EXIT

# Set the MODULE env variable if possible
defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"

# Prepare the cloud resources using Terraform
startCloudTestEnv "${MODULE_DIR}"

# Run tests
echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME"
pushd "${BEATS_PROJECT_NAME}" > /dev/null

mage build test

popd > /dev/null
171 changes: 158 additions & 13 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ arch_type="$(uname -m)"
GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
ONLY_DOCS=${ONLY_DOCS:-"true"}
OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
[ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")"
[ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")"
[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
Expand All @@ -17,6 +19,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"}
[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
[ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")"
[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"

metricbeat_changeset=(
"^metricbeat/.*"
Expand Down Expand Up @@ -81,6 +86,30 @@ packaging_changeset=(
".go-version"
)

case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-metricbeat")
BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]}
;;
"beats-libbeat")
BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]}
;;
"beats-packetbeat")
BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]}
;;
"beats-winlogbeat")
BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
;;
"beats-xpack-libbeat")
BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]}
;;
"beats-xpack-metricbeat")
BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
;;
*)
echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
;;
esac

check_and_set_beat_vars() {
if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then
BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove -
Expand All @@ -89,8 +118,8 @@ check_and_set_beat_vars() {
BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME}
TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}"
TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests"
TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests"
TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests"
declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset"
echo "Beats project name is $BEATS_XPACK_PROJECT_NAME"
mandatory_changeset=(
"${BEAT_CHANGESET_REFERENCE[@]}"
Expand All @@ -101,8 +130,8 @@ check_and_set_beat_vars() {
BEATS_GH_LABEL=${BEATS_PROJECT_NAME}
TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}"
TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests"
TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests"
TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests"
declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset"
echo "Beats project name is $BEATS_PROJECT_NAME"
mandatory_changeset=(
"${BEAT_CHANGESET_REFERENCE[@]}"
Expand All @@ -113,8 +142,10 @@ check_and_set_beat_vars() {
BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}"
BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos"
BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm"
BAETS_GH_MACOS_LABEL="macOS"
BAETS_GH_ARM_LABEL="arm"
BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud"
BEATS_GH_MACOS_LABEL="macOS"
BEATS_GH_ARM_LABEL="arm"
BEATS_GH_AWS_LABEL="aws"
}

with_docker_compose() {
Expand All @@ -127,6 +158,19 @@ with_docker_compose() {
docker-compose version
}

with_Terraform() {
echo "Setting up the Terraform environment..."
local path_to_file="${WORKSPACE}/terraform.zip"
create_workspace
check_platform_architeture
retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip"
unzip -q ${path_to_file} -d ${BIN}/
rm ${path_to_file}
chmod +x ${BIN}/terraform
export PATH="${BIN}:${PATH}"
terraform version
}

create_workspace() {
if [[ ! -d "${BIN}" ]]; then
mkdir -p "${BIN}"
Expand Down Expand Up @@ -168,6 +212,8 @@ with_mage() {
for pkg in "${install_packages[@]}"; do
go install "${pkg}@latest"
done
echo "Download modules to local cache"
retry 3 go mod download
}

with_go() {
Expand Down Expand Up @@ -283,9 +329,15 @@ are_paths_changed() {

are_changed_only_paths() {
local patterns=("${@}")
local changelist=()
local changed_files=$(git diff --name-only HEAD@{1} HEAD)
if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
local changed_files=($(git diff --name-only HEAD@{1} HEAD))
local matched_files=()
for pattern in "${patterns[@]}"; do
local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}"))
if [ "${#matched[@]}" -gt 0 ]; then
matched_files+=("${matched[@]}")
fi
done
if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then
return 0
fi
return 1
Expand All @@ -301,7 +353,7 @@ are_conditions_met_mandatory_tests() {
are_conditions_met_arm_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
return 0
fi
fi
Expand All @@ -311,8 +363,19 @@ are_conditions_met_arm_tests() {

are_conditions_met_macos_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
fi
fi
return 1
}

are_conditions_met_aws_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
fi
Expand All @@ -336,19 +399,101 @@ config_git() {
fi
}

defineModuleFromTheChangeSet() {
# This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module.
# For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files.
# This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module
local project_path=$1
local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g')
local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$"
local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")

if [[ "$project_path" == *"x-pack/"* ]]; then
local pattern=("$XPACK_MODULE_PATTERN")
else
local pattern=("$OSS_MODULE_PATTERN")
fi
local changed_modules=""
local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d)
for module_dir in $module_dirs; do
if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then
if [[ -z "$changed_modules" ]]; then
changed_modules=$(basename "$module_dir")
else
changed_modules+=",$(basename "$module_dir")"
fi
fi
done
if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
export MODULE="aws"
else
export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
# export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
fi
}

terraformInit() {
local dir=$1
echo "Terraform Init on $dir"
pushd "${dir}" > /dev/null
terraform init
popd > /dev/null
}

withAWS() {
# This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names.
export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY
export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
}

startCloudTestEnv() {
local dir=$1
withAWS
echo "--- Run docker-compose services for emulated cloud env"
docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
with_Terraform
terraformInit "$dir"
export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
export TF_VAR_CREATED_DATE=$(date +%s)
export TF_VAR_ENVIRONMENT="ci"
export TF_VAR_REPO="${REPO}"
pushd "${dir}" > /dev/null
terraform apply -auto-approve
popd > /dev/null
}

teardown() {
# Teardown resources after using them
echo "---Terraform Cleanup"
.ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK

echo "---Docker Compose Cleanup"
docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
}

unset_secrets () {
for var in $(printenv | sed 's;=.*;;' | sort); do
if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then
unset "$var"
fi
done
}

if ! are_changed_only_paths "${docs_changeset[@]}" ; then
ONLY_DOCS="false"
export ONLY_DOCS="false"
echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS."
else
echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS."
fi

if are_paths_changed "${go_mod_changeset[@]}" ; then
GO_MOD_CHANGES="true"
export GO_MOD_CHANGES="true"
fi

if are_paths_changed "${packaging_changeset[@]}" ; then
PACKAGING_CHANGES="true"
export PACKAGING_CHANGES="true"
fi

check_and_set_beat_vars
4 changes: 2 additions & 2 deletions .buildkite/scripts/generate_metricbeat_pipeline.sh
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ if are_conditions_met_packaging; then
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS: "${PACKAGING_PLATFORMS}"
- label: ":linux: Packaging ARM"
key: "packaging-arm"
Expand All @@ -159,7 +159,7 @@ if are_conditions_met_packaging; then
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
env:
PLATFORMS: "linux/arm64"
PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
PACKAGES: "docker"
YAML
Expand Down
6 changes: 3 additions & 3 deletions .buildkite/scripts/generate_packetbeat_pipeline.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ else
exit 0
fi

if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then
if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
cat >> $pipelineName <<- YAML
- group: "Extended Tests"
Expand Down Expand Up @@ -150,7 +150,7 @@ if are_conditions_met_packaging; then
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS: "${PACKAGING_PLATFORMS}"
- label: ":linux: Packaging ARM"
key: "packaging-arm"
Expand All @@ -160,7 +160,7 @@ if are_conditions_met_packaging; then
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
env:
PLATFORMS: "linux/arm64"
PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
PACKAGES: "docker"
YAML
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/generate_winlogbeat_pipeline.sh
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ if are_conditions_met_packaging; then
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS: "${PACKAGING_PLATFORMS}"
YAML
Expand Down
Loading

0 comments on commit 64ba153

Please sign in to comment.