diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index e0bc84b241e..a8def81f1fa 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -3,9 +3,8 @@ name: "beats-auditbeat"
 
 env:
   BEATS_PROJECT_NAME: "auditbeat"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -13,10 +12,11 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - input: "Input Parameters"
@@ -62,7 +62,7 @@ steps:
 
   - label: ":linux: Load dynamic auditbeat pipeline"
     key: "auditbeat-pipeline"
-    command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
+    command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh"
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh
similarity index 56%
rename from .buildkite/scripts/generate_auditbeat_pipeline.sh
rename to .buildkite/auditbeat/generate_auditbeat_pipeline.sh
index 0ff26cda5d3..56befb51d25 100644
--- a/.buildkite/scripts/generate_auditbeat_pipeline.sh
+++ b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh
@@ -6,19 +6,18 @@ set -euo pipefail
 
 pipelineName="pipeline.auditbeat-dynamic.yml"
 
-# TODO: steps: must be always included 
+# TODO: steps: must be always included
 echo "Add the mandatory and extended tests without additional conditions into the pipeline"
 if are_conditions_met_mandatory_tests; then
   cat > $pipelineName <<- YAML
 
 steps:
   - group: "Auditbeat Mandatory Testing"
-    key: "mandatory-tests"    
+    key: "mandatory-tests"
 
     steps:
       - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: linux/Unit Tests"
@@ -26,13 +25,10 @@ steps:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":rhel: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: rhel/Unit Tests"
@@ -40,43 +36,39 @@ steps:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":windows:-2016 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: windows 2016/Unit Tests"
+              context: "Auditbeat: windows-2016/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
-          disk_type: "pd-ssd"        
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"    
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":windows:-2022 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: windows 2022/Unit Tests"
+              context: "Auditbeat: windows-2022/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
-          disk_type: "pd-ssd"        
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"              
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":linux: Crosscompile"
         command:
-          - ".buildkite/auditbeat/scripts/crosscompile.sh"
+          - "make -C auditbeat crosscompile"
         env:
           GOX_FLAGS: "-arch amd64"
         notify:
@@ -94,22 +86,58 @@ fi
 
 echo "Check and add the Extended Tests into the pipeline"
 
-if are_conditions_met_arm_tests; then
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
   cat >> $pipelineName <<- YAML
+
   - group: "Extended Tests"
-    key: "extended-tests-arm"
+    key: "extended-tests"
     steps:
-      - label: ":arm: ARM64 Unit Tests"
-        key: "extended-arm64-unit-tests"
-        command: ".buildkite/scripts/unit_tests.sh"
+
+YAML
+fi
+
+if are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS Unit Tests"
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_ARM64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: ARM Unit tests"
+              context: "Auditbeat: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
 YAML
 fi
 
@@ -117,48 +145,49 @@ if are_conditions_met_win_tests; then
   cat >> $pipelineName <<- YAML
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
+
     steps:
-      - label: ":windows: Windows 2019 Unit Tests"
-        key: "extended-win-2019-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-2019-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-11-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: Windows 2019 Unit Tests"
-
-      - label: ":windows: Windows 10 Unit Tests"
-        key: "extended-win-10-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+              context: "Auditbeat: Win-11 Unit Tests"
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_10}"
+          image: "${IMAGE_WIN_11}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-10-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: Windows 10 Unit Tests"
-
-      - label: ":windows: Windows 11 Unit Tests"
-        key: "extended-win-11-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+              context: "Auditbeat: Win-10 Unit Tests"
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_11}"
+          image: "${IMAGE_WIN_10}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: Windows 11 Unit Tests"
 YAML
 fi
 
@@ -166,9 +195,10 @@ echo "Check and add the Packaging into the pipeline"
 if are_conditions_met_packaging; then
 cat >> $pipelineName <<- YAML
   - group: "Packaging"
-    key: "packaging"      
+    key: "packaging"
     depends_on:
       - "mandatory-tests"
+
     steps:
       - label: Package pipeline
         commands: ".buildkite/scripts/packaging/package-step.sh"
@@ -176,7 +206,6 @@ cat >> $pipelineName <<- YAML
           - github_commit_status:
               context: "Auditbeat: Packaging"
 
-
 YAML
 fi
 
diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh
deleted file mode 100755
index da8452d5380..00000000000
--- a/.buildkite/auditbeat/scripts/crosscompile.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-echo "--- Executing Crosscompile"
-make -C auditbeat crosscompile
diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh
deleted file mode 100755
index 4b8e86243c9..00000000000
--- a/.buildkite/auditbeat/scripts/unit-tests.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-echo "--- Running Unit Tests"
-sudo chmod -R go-w auditbeat/
-
-umask 0022
-mage -d auditbeat build unitTest
diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
index 34321b61161..246aac9f80a 100644
--- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
+++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -1,5 +1,65 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: "Checks"
+    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
+    notify:
+      - github_commit_status:
+          context: "Deploy K8S/Checks"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.29.0"
+    key: "k8s-test-129"
+    env:
+      K8S_VERSION: "v1.29.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.28.0"
+    key: "k8s-test-128"
+    env:
+      K8S_VERSION: "v1.28.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.27.3"
+    key: "k8s-test-1273"
+    env:
+      K8S_VERSION: "v1.27.3"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.26.6"
+    key: "k8s-test-1266"
+    env:
+      K8S_VERSION: "v1.26.6"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh
new file mode 100755
index 00000000000..8c399d2de37
--- /dev/null
+++ b/.buildkite/deploy/kubernetes/scripts/install-kind.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MSG="environment variable missing."
+KIND_VERSION=${KIND_VERSION:?$MSG}
+KIND_BINARY="${BIN}/kind"
+
+if command -v kind
+then
+    set +e
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
+    then
+        echo "--- Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing Kind"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
+    chmod +x "${KIND_BINARY}"
+    echo "Kind installed: ${KIND_VERSION}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KIND_BINARY}" ] ; then
+        rm "${KIND_BINARY}"
+    fi
+    exit 1
+fi
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh
new file mode 100755
index 00000000000..7f6c75bf3b9
--- /dev/null
+++ b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+KUBECTL_BINARY="${BIN}/kubectl"
+
+if command -v kubectl
+then
+    set +e
+    echo "Found kubectl. Checking version.."
+    FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
+    if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
+    then
+        echo "Kubectl Versions match. No need to install kubectl. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing kubectl"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KUBECTL_BINARY}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
+    chmod +x "${KUBECTL_BINARY}"
+    echo "Current K8S Version: ${K8S_VERSION}"
+    echo "Kubectl installed: ${KUBECTL_BINARY}"
+else
+    echo "--- Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KUBECTL_BINARY}" ] ; then
+        rm "${KUBECTL_BINARY}"
+    fi
+    exit 1
+fi
+
diff --git a/.buildkite/deploy/kubernetes/scripts/kind-setup.sh b/.buildkite/deploy/kubernetes/scripts/kind-setup.sh
new file mode 100755
index 00000000000..ca46cdb0fd8
--- /dev/null
+++ b/.buildkite/deploy/kubernetes/scripts/kind-setup.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+echo "Cluster info: "
+kubectl cluster-info
diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh
new file mode 100755
index 00000000000..4c9120a2d90
--- /dev/null
+++ b/.buildkite/deploy/kubernetes/scripts/make.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "--- Checking K8S"
+make -C deploy/kubernetes all
+make check-no-changes
diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
new file mode 100755
index 00000000000..3656318bd64
--- /dev/null
+++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+echo "--- Installing kind & kubectl"
+retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh
+retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh
+
+echo "--- Setting up kind"
+max_retries=3
+timeout=5
+retries=0
+
+while true; do
+  echo "Creating cluster"
+  script_output=$(.buildkite/deploy/kubernetes/scripts/kind-setup.sh 2>&1)
+  exit_code=$?
+
+  echo "Script Output: $script_output"
+
+  if [ $exit_code -eq 0 ]; then
+    break
+  else
+    retries=$((retries + 1))
+
+    if [ $retries -gt $max_retries ]; then
+      kind delete cluster
+      echo "Kind setup FAILED: $script_output"
+      exit 1
+    fi
+
+    kind delete cluster
+
+    sleep_time=$((timeout * retries))
+    echo "Retry #$retries failed. Retrying after ${sleep_time}s..."
+    sleep $sleep_time
+  fi
+done
diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh
index f28658a107d..b30b26c3e8a 100644
--- a/.buildkite/env-scripts/env.sh
+++ b/.buildkite/env-scripts/env.sh
@@ -30,6 +30,9 @@ ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
 PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
 GO_MOD_CHANGES=$(changeset_applies "^go.mod")
 
+KIND_VERSION="v0.20.0"
+KUBECONFIG="${WORKSPACE}/kubecfg"
+
 export REPO
 export WORKSPACE
 export BIN
@@ -51,3 +54,8 @@ export DOCKER_REGISTRY
 export ONLY_DOCS
 export PACKAGING_CHANGES
 export GO_MOD_CHANGES
+
+export KIND_VERSION
+export KUBECONFIG
+
+add_bin_path
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index d469cdc0118..b0737eb60b2 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -5,17 +5,17 @@ env:
   BEATS_PROJECT_NAME: "filebeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - input: "Input Parameters"
@@ -61,7 +61,7 @@ steps:
 
   - label: ":linux: Load dynamic filebeat pipeline"
     key: "filebeat-pipeline"
-    command: ".buildkite/scripts/generate_filebeat_pipeline.sh"
+    command: ".buildkite/filebeat/generate_filebeat_pipeline.sh"
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
diff --git a/.buildkite/filebeat/generate_filebeat_pipeline.sh b/.buildkite/filebeat/generate_filebeat_pipeline.sh
new file mode 100644
index 00000000000..aa0a03eadc1
--- /dev/null
+++ b/.buildkite/filebeat/generate_filebeat_pipeline.sh
@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.filebeat-dynamic.yml"
+
+# TODO: steps: must be always included
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+  - group: "Filebeat Mandatory Testing"
+    key: "mandatory-tests"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: linux/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":ubuntu: Go Integration Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Go Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":ubuntu: Python Integration Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Python Integration Tests"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-2016 Unit Tests"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: windows-2016/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-2022 Unit Tests"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: windows-2022/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+echo "Check and add the Extended Tests into the pipeline"
+
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+YAML
+fi
+
+if are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+fi
+
+if are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+fi
+
+if are_conditions_met_win_tests; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "windows-extended-2019"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-extended-11"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Win-11 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-extended-10"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Win-10 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+cat >> $pipelineName <<- YAML
+
+  - group: "Packaging"
+    key: "packaging"
+    depends_on:
+      - "mandatory-tests"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/scripts/packaging/package-step.sh"
+        notify:
+        - github_commit_status:
+            context: "Filebeat: Packaging"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName
diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh
deleted file mode 100755
index 6de39ff8817..00000000000
--- a/.buildkite/filebeat/scripts/integration-gotests.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-echo "--- Executing Integration Tests"
-sudo chmod -R go-w filebeat/
-
-cd filebeat
-umask 0022
-mage goIntegTest
diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh
deleted file mode 100755
index 9aff8695c35..00000000000
--- a/.buildkite/filebeat/scripts/integration-pytests.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-echo "--- Executing Integration Tests"
-sudo chmod -R go-w filebeat/
-
-cd filebeat
-umask 0022
-mage pythonIntegTest
diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh
deleted file mode 100755
index 4365f9d8d51..00000000000
--- a/.buildkite/filebeat/scripts/package-step.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-source .buildkite/env-scripts/util.sh
-
-changeset="^filebeat/
-^go.mod
-^pytest.ini
-^dev-tools/
-^libbeat/
-^testing/
-^\.buildkite/filebeat/"
-
-if are_files_changed "$changeset"; then
-  bk_pipeline=$(cat <<-YAML
-    steps:
-      - label: ":ubuntu: Packaging Linux X86"
-        key: "package-linux-x86"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
-        command:
-          - ".buildkite/filebeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Packaging: Linux X86"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-
-      - label: ":linux: Packaging Linux ARM"
-        key: "package-linux-arm"
-        env:
-          PLATFORMS: "linux/arm64"
-          PACKAGES: "docker"
-        command:
-          - ".buildkite/filebeat/scripts/package.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Packaging: ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
-YAML
-)
-  echo "${bk_pipeline}" | buildkite-agent pipeline upload
-else
-  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
-  exit 0
-fi
diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh
deleted file mode 100755
index 2efb6b1ff8e..00000000000
--- a/.buildkite/filebeat/scripts/unit-tests.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-echo "--- Executing Unit Tests"
-sudo chmod -R go-w filebeat/
-
-umask 0022
-mage -d filebeat unitTest
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 2b5f6195f45..e63fb1c60e2 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -3,16 +3,19 @@
 env:
   BEATS_PROJECT_NAME: "heartbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  IMAGE_WIN_2016: "family/core-windows-2016"
-  IMAGE_WIN_2019: "family/core-windows-2019"
-  IMAGE_WIN_2022: "family/core-windows-2022"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
+  IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
+  IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
+  IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - group: "Heartbeat Mandatory Testing"
@@ -20,48 +23,64 @@ steps:
     if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
 
     steps:
-      - label: ":linux: Unit Tests / {{matrix.image}}"
-        command:
-          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+      - label: ":ubuntu: Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Heartbeat: linux/Unit Tests"
+              context: "Heartbeat: ubuntu/Unit Tests"
         agents:
           provider: "gcp"
-          image: "{{matrix.image}}"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":rhel:-9 Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: rhel-9/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_UBUNTU_X86_64}"
-              - "${IMAGE_RHEL9}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
-      - label: ":windows: Unit Tests / {{matrix.image}}"
-        command:
-          - ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2016 Unit Test"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: windows-2016/Unit Tests"
+        agents:
+          provider: "gcp"
+          image:  "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows:-2022 Unit Test"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Heartbeat: windows/Unit Tests"
+              context: "Heartbeat: windows-2022/Unit Tests"
         agents:
           provider: "gcp"
-          image: "{{matrix.image}}"
+          image: "${IMAGE_WIN_2022}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
       - label: ":ubuntu: Go Integration Tests"
-        command:
-          - ".buildkite/heartbeat/scripts/integration-gotests.sh"
+        command:  "cd ${BEATS_PROJECT_NAME} && mage goIntegTest"
         notify:
           - github_commit_status:
               context: "Heartbeat: Go Integration Tests"
@@ -74,8 +93,7 @@ steps:
           - "heartbeat/build/*.json"
 
       - label: ":ubuntu: Python Integration Tests"
-        command:
-          - ".buildkite/heartbeat/scripts/integration-pytests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest"
         notify:
           - github_commit_status:
               context: "Heartbeat: Python Integration Tests"
@@ -95,11 +113,10 @@ steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Heartbeat/Extended: Unit Tests ARM"
+              context: "Heartbeat: Unit Tests ARM"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -109,27 +126,38 @@ steps:
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Heartbeat/Extended: MacOS Unit Tests"
+              context: "Heartbeat: MacOS Unit Tests"
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths: "heartbeat/build/*.xml"
 
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-extended-arm"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "heartbeat/build/*.xml"
+
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
     if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-extended-2019"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Heartbeat/Extended: Win-2019 Unit Tests"
+              context: "Heartbeat: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -139,6 +167,36 @@ steps:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
 
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-extended-11"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-11 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-extended-10"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-10 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index e03ade43af1..73aec305150 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,44 +2,29 @@
 
 set -euo pipefail
 
+source .buildkite/env-scripts/util.sh
+
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
-PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-retry() {
-  local retries=$1
-  shift
-  local count=0
-  until "$@"; do
-    exit=$?
-    wait=$((2 ** count))
-    count=$((count + 1))
-    if [ $count -lt "$retries" ]; then
-      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-      sleep $wait
-    else
-      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-      return $exit
-    fi
-  done
-  return 0
-}
-
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
   source .buildkite/env-scripts/env.sh
-  source .buildkite/env-scripts/util.sh
-
-  if [[ -z "${GO_VERSION-""}" ]]; then
-    export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
-  fi
 
+  # Images with prefix "platform-ingest-beats-*" has Go pre-setup.
+  # Image itself takes care of Go version download/installation based on .go-version file
   if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    ulimit -Sn 30000
+    if [[ -z "${GO_VERSION-""}" ]]; then
+      GO_VERSION=$(cat "${WORKSPACE}/.go-version")
+      export GO_VERSION
+    fi
+
+    # To prevent "OSError: [Errno 24] Too many open files"
+    ulimit -Sn 150000
+    echo "--- Ulimit: $(ulimit)"
 
     echo "--- Setting up environment"
     add_bin_path
@@ -47,10 +32,20 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi
     with_mage
   fi
 
+  if [[ "$BUILDKITE_STEP_KEY" == windows* ]]; then
+    git config core.autocrlf false
+    git rm --quiet --cached -r .
+    git reset --quiet --hard
+  fi
+
   if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    export GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+    DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+    GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+
+    export DOCKER_USERNAME_SECRET
+    export DOCKER_PASSWORD_SECRET
+    export GITHUB_TOKEN_SECRET
 
     docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
 
@@ -60,6 +55,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi
     git config user.name "$github_username"
     git config user.email "$github_email"
   fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then
+    .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+  fi
 fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(
@@ -76,8 +75,8 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "beats-xpack-winlogbeat"
   "beats-xpack-dockerlogbeat"
   "beats-xpack-auditbeat"
-  "beats-xpack-filebeat" 
-  "beats-xpack-heartbeat" 
+  "beats-xpack-filebeat"
+  "beats-xpack-heartbeat"
   "beats-xpack-osquerybeat"
   )
 
@@ -90,16 +89,16 @@ done
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then
-    BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    BEATS_AWS_SECRET_KEY=$(retry_with_count 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export BEATS_AWS_SECRET_KEY
-    BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+    BEATS_AWS_ACCESS_KEY=$(retry_with_count 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export BEATS_AWS_ACCESS_KEY
   fi
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
-    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
   fi
 fi
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index dcd4dfbf208..9088d0dce39 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -26,7 +26,7 @@
             "build_on_commit": true,
             "build_on_comment": true,
             "trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
-            "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/package filebeat$",
+            "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
             "skip_ci_labels": [ ],
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
@@ -57,8 +57,8 @@
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$",
-            "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$",
+            "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
+            "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
             "skip_ci_labels": [  ],
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
@@ -73,8 +73,8 @@
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$",
-            "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$",
+            "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
+            "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$",
             "skip_ci_labels": [  ],
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh
deleted file mode 100644
index 63108d5aacd..00000000000
--- a/.buildkite/scripts/generate_filebeat_pipeline.sh
+++ /dev/null
@@ -1,168 +0,0 @@
-#!/usr/bin/env bash
-
-source .buildkite/scripts/common.sh
-
-set -euo pipefail
-
-pipelineName="pipeline.filebeat-dynamic.yml"
-
-# TODO: steps: must be always included 
-echo "Add the mandatory and extended tests without additional conditions into the pipeline"
-if are_conditions_met_mandatory_tests; then
-  cat > $pipelineName <<- YAML
-
-
-steps:
-  - group: "Filebeat Mandatory Testing"
-    key: "mandatory-tests"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-
-    steps:
-      - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: linux/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Go Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-gotests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Go Integration Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Python Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-pytests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Python Integration Tests"
-        agents:
-          provider: gcp
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":windows:-2016 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: windows/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":windows:-2022 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: windows 2022/Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"        
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-YAML
-else
-  echo "The conditions don't match to requirements for generating pipeline steps."
-  exit 0
-fi
-
-echo "Check and add the Extended Tests into the pipeline"
-
-if are_conditions_met_arm_tests; then
-  cat >> $pipelineName <<- YAML
-  - group: "Extended Tests: ARM"
-      key: "extended-tests-arm"
-      steps:
-      - label: ":linux: ARM64 Unit Tests"
-        key: "arm-extended"        
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Unit Tests ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "filebeat/build/*.xml"
-
-YAML
-fi
-
-if are_conditions_met_win_tests; then
-  cat >> $pipelineName <<- YAML
-  - group: "Windows Extended Testing"
-    key: "extended-tests-win"
-    steps:
-    - label: ":windows: Win 2019 Unit Tests"
-      key: "win-extended-2019"
-      command: ".buildkite/scripts/win_unit_tests.ps1"
-      notify:
-        - github_commit_status:
-            context: "Filebeat/Extended: Win-2019 Unit Tests"
-      agents:
-        provider: "gcp"
-        image: "${IMAGE_WIN_2019}"
-        machine_type: "${GCP_WIN_MACHINE_TYPE}"
-        disk_size: 200
-        disk_type: "pd-ssd"
-      artifact_paths:
-        - "filebeat/build/*.xml"
-        - "filebeat/build/*.json"
-YAML
-fi
-
-echo "Check and add the Packaging into the pipeline"
-if are_conditions_met_packaging; then
-cat >> $pipelineName <<- YAML
-  - group: "Packaging"
-    key: "packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
-
-    steps:
-      - label: Package pipeline
-        commands: ".buildkite/scripts/packaging/package-step.sh"
-        notify:
-        - github_commit_status:
-            context: "Filebeat: Packaging"
-
-YAML
-fi
-
-echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
-cat $pipelineName
-
-echo "--- Loading dynamic steps"
-buildkite-agent pipeline upload $pipelineName
diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh
index 61b2e2f5d90..59cf1660436 100755
--- a/.buildkite/scripts/setenv.sh
+++ b/.buildkite/scripts/setenv.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
+
 REPO="beats"
 TMP_FOLDER="tmp.${REPO}"
 DOCKER_REGISTRY="docker.elastic.co"
diff --git a/auditbeat/module/file_integrity/fileinfo_test.go b/auditbeat/module/file_integrity/fileinfo_test.go
index ea3a16b65ed..ce9d62d1087 100644
--- a/auditbeat/module/file_integrity/fileinfo_test.go
+++ b/auditbeat/module/file_integrity/fileinfo_test.go
@@ -30,6 +30,9 @@ import (
 )
 
 func TestNewMetadata(t *testing.T) {
+	// Can be removed after https://github.com/elastic/beats/issues/37701 is solved
+	skipOnBuildkiteDarwin(t, "Group check")
+
 	f, err := ioutil.TempFile("", "metadata")
 	if err != nil {
 		t.Fatal(err)
@@ -96,6 +99,9 @@ func TestNewMetadata(t *testing.T) {
 }
 
 func TestSetUIDSetGIDBits(t *testing.T) {
+	// Can be removed after https://github.com/elastic/beats/issues/37701 is solved
+	skipOnBuildkiteDarwin(t, "Wheel permission issue")
+
 	f, err := ioutil.TempFile("", "setuid")
 	if err != nil {
 		t.Fatal(err)
@@ -150,3 +156,9 @@ func TestSetUIDSetGIDBits(t *testing.T) {
 		assert.Equal(t, flags&os.ModeSetgid != 0, meta.SetGID)
 	}
 }
+
+func skipOnBuildkiteDarwin(t testing.TB, reason string) {
+	if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "darwin" {
+		t.Skip("Skip test on Buildkite MacOS: Wheel permission while expected staff")
+	}
+}
diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go
index b4372aa8ba7..386c34419ec 100644
--- a/auditbeat/module/file_integrity/metricset_test.go
+++ b/auditbeat/module/file_integrity/metricset_test.go
@@ -66,7 +66,11 @@ func TestData(t *testing.T) {
 
 func TestActions(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
 	skipOnBuildkiteWindows(t)
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
+	skipOnBuildkiteDarwinArm(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -177,7 +181,11 @@ func TestActions(t *testing.T) {
 
 func TestExcludedFiles(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
 	skipOnBuildkiteWindows(t)
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
+	skipOnBuildkiteDarwinArm(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -234,7 +242,11 @@ func TestExcludedFiles(t *testing.T) {
 
 func TestIncludedExcludedFiles(t *testing.T) {
 	skipOnCIForDarwinAMD64(t)
+
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3016 is solved
 	skipOnBuildkiteWindows(t)
+	// Can be removed after https://github.com/elastic/ingest-dev/issues/3076 is solved
+	skipOnBuildkiteDarwinArm(t)
 
 	defer abtest.SetupDataDir(t)()
 
@@ -1001,3 +1013,9 @@ func skipOnBuildkiteWindows(t testing.TB) {
 		t.Skip("Skip on Buildkite Windows: Shortened TMP problem")
 	}
 }
+
+func skipOnBuildkiteDarwinArm(t testing.TB) {
+	if os.Getenv("BUILDKITE") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+		t.Skip("Skip test on Buldkite: unexpected path error")
+	}
+}