diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 0ac7c51099c..282fb5a0085 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,6 +2,27 @@
 
 set -euo pipefail
 
+AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/util.sh
@@ -12,10 +33,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"
     exit 0
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then
+  BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  export BEATS_AWS_SECRET_KEY
+  BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  export BEATS_AWS_ACCESS_KEY
+fi
diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh
new file mode 100755
index 00000000000..9e7ce95be66
--- /dev/null
+++ b/.buildkite/scripts/cloud_tests.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# What Terraform Module will run
+export MODULE_DIR="x-pack/metricbeat/module/aws"
+
+source .buildkite/scripts/install_tools.sh
+
+set -euo pipefail
+
+trap 'teardown || true; unset_secrets' EXIT
+
+# Set the MODULE env variable if possible
+defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
+
+# Prepare the cloud resources using Terraform
+startCloudTestEnv "${MODULE_DIR}"
+
+# Run tests
+echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME"
+pushd "${BEATS_PROJECT_NAME}" > /dev/null
+
+mage build test
+
+popd > /dev/null
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index b797ec718aa..ebb15c937dd 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -9,6 +9,8 @@ arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")"
 [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")"
 [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
@@ -17,6 +19,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"}
 [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
 [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")"
 [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
+[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
+[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
+[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
@@ -81,6 +86,30 @@ packaging_changeset=(
   ".go-version"
   )
 
+case "${BUILDKITE_PIPELINE_SLUG}" in
+  "beats-metricbeat")
+    BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]}
+    ;;
+  "beats-libbeat")
+    BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]}
+    ;;
+  "beats-packetbeat")
+    BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]}
+    ;;
+  "beats-winlogbeat")
+    BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
+    ;;
+  "beats-xpack-libbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]}
+    ;;
+  "beats-xpack-metricbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
+    ;;
+  *)
+  echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
+  ;;
+esac
+
 check_and_set_beat_vars() {
   if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then
     BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/}              #remove -
@@ -89,8 +118,8 @@ check_and_set_beat_vars() {
     BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME}
     TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}"
     TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests"
+    TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests"
     TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests"
-    declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset"
     echo "Beats project name is $BEATS_XPACK_PROJECT_NAME"
     mandatory_changeset=(
       "${BEAT_CHANGESET_REFERENCE[@]}"
@@ -101,8 +130,8 @@ check_and_set_beat_vars() {
     BEATS_GH_LABEL=${BEATS_PROJECT_NAME}
     TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}"
     TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests"
+    TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests"
     TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests"
-    declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset"
     echo "Beats project name is $BEATS_PROJECT_NAME"
     mandatory_changeset=(
       "${BEAT_CHANGESET_REFERENCE[@]}"
@@ -113,8 +142,10 @@ check_and_set_beat_vars() {
   BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}"
   BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos"
   BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm"
-  BAETS_GH_MACOS_LABEL="macOS"
-  BAETS_GH_ARM_LABEL="arm"
+  BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud"
+  BEATS_GH_MACOS_LABEL="macOS"
+  BEATS_GH_ARM_LABEL="arm"
+  BEATS_GH_AWS_LABEL="aws"
 }
 
 with_docker_compose() {
@@ -127,6 +158,19 @@ with_docker_compose() {
   docker-compose version
 }
 
+with_Terraform() {
+    echo "Setting up the Terraform environment..."
+    local path_to_file="${WORKSPACE}/terraform.zip"
+    create_workspace
+    check_platform_architeture
+    retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip"
+    unzip -q ${path_to_file} -d ${BIN}/
+    rm ${path_to_file}
+    chmod +x ${BIN}/terraform
+    export PATH="${BIN}:${PATH}"
+    terraform version
+}
+
 create_workspace() {
   if [[ ! -d "${BIN}" ]]; then
     mkdir -p "${BIN}"
@@ -168,6 +212,8 @@ with_mage() {
   for pkg in "${install_packages[@]}"; do
     go install "${pkg}@latest"
   done
+  echo "Download modules to local cache"
+  retry 3 go mod download
 }
 
 with_go() {
@@ -283,9 +329,15 @@ are_paths_changed() {
 
 are_changed_only_paths() {
   local patterns=("${@}")
-  local changelist=()
-  local changed_files=$(git diff --name-only HEAD@{1} HEAD)
-  if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
+  local changed_files=($(git diff --name-only HEAD@{1} HEAD))
+  local matched_files=()
+  for pattern in "${patterns[@]}"; do
+    local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}"))
+    if [ "${#matched[@]}" -gt 0 ]; then
+      matched_files+=("${matched[@]}")
+    fi
+  done
+  if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then
     return 0
   fi
   return 1
@@ -301,7 +353,7 @@ are_conditions_met_mandatory_tests() {
 are_conditions_met_arm_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
         return 0
       fi
     fi
@@ -311,8 +363,19 @@ are_conditions_met_arm_tests() {
 
 are_conditions_met_macos_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+        return 0
+      fi
+    fi
+  fi
+  return 1
+}
+
+are_conditions_met_aws_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
         return 0
       fi
     fi
@@ -336,19 +399,101 @@ config_git() {
   fi
 }
 
+defineModuleFromTheChangeSet() {
+  # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module.
+  # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files.
+  # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module
+  local project_path=$1
+  local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g')
+  local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$"
+  local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")
+
+  if [[ "$project_path" == *"x-pack/"* ]]; then
+    local pattern=("$XPACK_MODULE_PATTERN")
+  else
+    local pattern=("$OSS_MODULE_PATTERN")
+  fi
+  local changed_modules=""
+  local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d)
+  for module_dir in $module_dirs; do
+    if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then
+      if [[ -z "$changed_modules" ]]; then
+        changed_modules=$(basename "$module_dir")
+      else
+        changed_modules+=",$(basename "$module_dir")"
+      fi
+    fi
+  done
+  if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+    export MODULE="aws"
+  else
+    export MODULE="${changed_modules}"  # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+  # export MODULE="${changed_modules}"     # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved
+  fi
+}
+
+terraformInit() {
+  local dir=$1
+  echo "Terraform Init on $dir"
+  pushd "${dir}" > /dev/null
+  terraform init
+  popd > /dev/null
+}
+
+withAWS() {
+  # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names.
+  export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
+  export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY
+  export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
+}
+
+startCloudTestEnv() {
+  local dir=$1
+  withAWS
+  echo "--- Run docker-compose services for emulated cloud env"
+  docker-compose -f .ci/jobs/docker-compose.yml up -d                     #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  with_Terraform
+  terraformInit "$dir"
+  export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
+  export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
+  export TF_VAR_CREATED_DATE=$(date +%s)
+  export TF_VAR_ENVIRONMENT="ci"
+  export TF_VAR_REPO="${REPO}"
+  pushd "${dir}" > /dev/null
+  terraform apply -auto-approve
+  popd > /dev/null
+}
+
+teardown() {
+  # Teardown resources after using them
+  echo "---Terraform Cleanup"
+  .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}"              #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+
+  echo "---Docker Compose Cleanup"
+  docker-compose -f .ci/jobs/docker-compose.yml down -v         #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then
+      unset "$var"
+    fi
+  done
+}
+
 if ! are_changed_only_paths "${docs_changeset[@]}" ; then
-  ONLY_DOCS="false"
+  export ONLY_DOCS="false"
   echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS."
 else
   echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS."
 fi
 
 if are_paths_changed "${go_mod_changeset[@]}" ; then
-  GO_MOD_CHANGES="true"
+  export GO_MOD_CHANGES="true"
 fi
 
 if are_paths_changed "${packaging_changeset[@]}" ; then
-  PACKAGING_CHANGES="true"
+  export PACKAGING_CHANGES="true"
 fi
 
 check_and_set_beat_vars
diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh
index e91896eb70c..477f8fb25a3 100755
--- a/.buildkite/scripts/generate_metricbeat_pipeline.sh
+++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh
@@ -149,7 +149,7 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
 
       - label: ":linux: Packaging ARM"
         key: "packaging-arm"
@@ -159,7 +159,7 @@ if are_conditions_met_packaging; then
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
 
 YAML
diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh
index 1186fd49c01..eb314248133 100755
--- a/.buildkite/scripts/generate_packetbeat_pipeline.sh
+++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh
@@ -92,7 +92,7 @@ else
   exit 0
 fi
 
-if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"
@@ -150,7 +150,11 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
+<<<<<<< HEAD
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
+=======
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
+>>>>>>> 843010c1cb (migrate xpack-metricbeat (#38081))
 
       - label: ":linux: Packaging ARM"
         key: "packaging-arm"
@@ -160,7 +164,7 @@ if are_conditions_met_packaging; then
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
 
 YAML
diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh
index 1eb1b459c92..ce812016e79 100755
--- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh
+++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh
@@ -96,7 +96,7 @@ if are_conditions_met_packaging; then
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
 
 
 YAML
diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh
new file mode 100755
index 00000000000..c9c65a5e757
--- /dev/null
+++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh
@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.xpack-metricbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":go: Go Integration Tests"
+        key: "mandatory-int-test"
+        command: ".buildkite/scripts/go_int_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":python: Python Integration Tests"
+        key: "mandatory-python-int-test"
+        command: ".buildkite/scripts/py_int_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "mandatory-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h
+  - group: "Extended Windows Tests"
+    key: "extended-win-tests"
+    steps:
+      - label: ":windows: Windows 10 Unit Tests"
+        key: "extended-win-10-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Windows 11 Unit Tests"
+        key: "extended-win-11-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "extended-win-2019-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+#TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved
+if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then
+# if [[ are_conditions_met_macos_tests ]]; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+
+YAML
+fi
+
+if  are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "extended-macos-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if  are_conditions_met_aws_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: Cloud Tests"
+        key: "extended-cloud-test"
+        command: ".buildkite/scripts/cloud_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${DEFAULT_UBUNTU_X86_64_IMAGE}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+  cat >> $pipelineName <<- YAML
+
+  - wait: ~
+    depends_on:
+      - step: "mandatory-tests"
+        allow_failure: false
+
+  - group: "Packaging"    # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        env:
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PACKAGES: "docker"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName
diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh
index 25121de212f..974886061d4 100755
--- a/.buildkite/scripts/setenv.sh
+++ b/.buildkite/scripts/setenv.sh
@@ -1,13 +1,20 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-
+REPO="beats"
+TMP_FOLDER="tmp.${REPO}"
+DOCKER_REGISTRY="docker.elastic.co"
 SETUP_GVM_VERSION="v0.5.1"
 DOCKER_COMPOSE_VERSION="1.21.0"
 DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
 SETUP_WIN_PYTHON_VERSION="3.11.0"
 NMAP_WIN_VERSION="7.12"           # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
 GO_VERSION=$(cat .go-version)
+ASDF_MAGE_VERSION="1.15.0"
+ASDF_TERRAFORM_VERSION="1.0.2"
+PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+PACKAGING_ARM_PLATFORMS="linux/arm64"
+AWS_REGION="eu-central-1"
 
 export SETUP_GVM_VERSION
 export DOCKER_COMPOSE_VERSION
@@ -15,6 +22,14 @@ export DOCKER_COMPOSE_VERSION_AARCH64
 export SETUP_WIN_PYTHON_VERSION
 export NMAP_WIN_VERSION
 export GO_VERSION
+export ASDF_MAGE_VERSION
+export PACKAGING_PLATFORMS
+export PACKAGING_ARM_PLATFORMS
+export REPO
+export TMP_FOLDER
+export DOCKER_REGISTRY
+export ASDF_TERRAFORM_VERSION
+export AWS_REGION
 
 exportVars() {
   local platform_type="$(uname)"
@@ -41,10 +56,10 @@ exportVars() {
   fi
 }
 
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
   exportVars
   export RACE_DETECTOR="true"
   export TEST_COVERAGE="true"
   export DOCKER_PULL="0"
+  export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle"
 fi
diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
index 01695fa4fb6..0c745b1a09d 100644
--- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml
@@ -20,7 +20,7 @@ steps:
   - input: "Input Parameters"
     key: "input-run-all-stages"
     fields:
-    - select: "Packetbeat - run_xpack_libbeat"
+    - select: "Libbeat - run_xpack_libbeat"
       key: "run_xpack_libbeat"
       options:
         - label: "True"
@@ -28,7 +28,7 @@ steps:
         - label: "False"
           value: "false"
       default: "false"
-    - select: "Packetbeat - run_xpack_libbeat_arm_tests"
+    - select: "Libbeat - run_xpack_libbeat_arm_tests"
       key: "run_xpack_libbeat_arm_tests"
       options:
         - label: "True"
@@ -42,8 +42,8 @@ steps:
     if: "build.source == 'ui'"
     allow_dependency_failure: false
 
-  - label: ":linux: Load dynamic packetbeat pipeline"
-    key: "packetbeat-pipeline"
+  - label: ":linux: Load dynamic x-pack libbeat pipeline"
+    key: "libbeat-pipeline"
     command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh"
     notify:
       - github_commit_status:
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 34321b61161..81756564919 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -1,5 +1,61 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: "beats-xpack-metricbeat"
+
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  BEATS_PROJECT_NAME: "x-pack/metricbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "force-run-stages"
+    fields:
+    - select: "Metricbeat - run_xpack_metricbeat"
+      key: "run_xpack_metricbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Metricbeat - run_xpack_metricbeat_macos_tests"
+      key: "run_xpack_metricbeat_macos_tests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Metricbeat - run_xpack_metricbeat_aws_tests"
+      key: "run_xpack_metricbeat_aws_tests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic x-pack metricbeat pipeline"
+    key: "metricbeat-pipeline"
+    command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh"
+    notify:
+      - github_commit_status:
+          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"