added image tag and push

.buildkite/env-scripts/env.sh

@@ -1,13 +1,34 @@
 #!/usr/bin/env bash
 
+source .buildkite/env-scripts/util.sh
+
+docs_changeset="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+packaging_changeset="^dev-tools/packaging/
+^.go-version"
+
 SETUP_GVM_VERSION="v0.5.1"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 
+# Env vars for Packaging stage
+DOCKER_REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION=$(make get-version | tr -d '\n')
+ONLY_DOCS=$(changeset_applies "$docs_changeset")
+PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
 export SETUP_GVM_VERSION
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
+export PACKAGING_CHANGES
+export ONLY_DOCS
+export GO_MOD_CHANGES
+export DOCKER_REGISTRY
+export SNAPSHOT
+export VERSION

.buildkite/env-scripts/linux-env.sh

@@ -18,7 +18,10 @@ if [[ $PLATFORM_TYPE == "Linux" ]]; then
   sudo apt-get install -y python3-venv
 fi
 
-echo ":: Setting up environment ::"
-add_bin_path
-with_go
-with_mage
+# Remove this code once beats specific ARM agent is set up
+if [[ $HW_TYPE == "aarch64" ]]; then
+  echo ":: Setting up environment ::"
+  add_bin_path
+  with_go
+  with_mage
+fi

.buildkite/env-scripts/util.sh

@@ -89,3 +89,40 @@ are_files_changed() {
     return 1;
   fi
 }
+
+changeset_applies() {
+  local changeset=$1
+  if are_files_changed "$changeset"; then
+    echo true
+  else
+    echo false
+  fi
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    unset GOOGLE_APPLICATION_CREDENTIALS
+    cleanup
+  fi
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}

.buildkite/filebeat/filebeat-pipeline.yml

@@ -7,136 +7,136 @@ env:
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_13: "generic-13-ventura-x64"
+  SETUP_MAGE_VERSION: "1.14.0"
+  ASDF_MAGE_VERSION: "1.14.0"
 
 steps:
-  - group: "Filebeat Mandatory Testing"
-    key: "mandatory-tests"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-
-    steps:
-      - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Go Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-gotests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Integration Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":ubuntu: Python Integration Tests"
-        command:
-          - ".buildkite/filebeat/scripts/integration-pytests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Python Integration Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-      - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat: Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        matrix:
-          setup:
-            image:
-              - "${IMAGE_WIN_2016}"
-              - "${IMAGE_WIN_2022}"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
-
-  - group: "Extended Testing"
-    key: "extended-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
-
-    steps:
-      - label: ":linux: ARM64 Unit Tests"
-        key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Unit Tests ARM"
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
-        artifact_paths: "filebeat/build/*.xml"
-
-      - label: ":mac: MacOS Unit Tests"
-        key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command:
-          - ".buildkite/filebeat/scripts/unit-tests.sh"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: MacOS Unit Tests"
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_13}"
-        artifact_paths: "filebeat/build/*.xml"
-
-  - group: "Windows Extended Testing"
-    key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-
-    steps:
-      - label: ":windows: Win 2019 Unit Tests"
-        key: "win-extended-2019"
-        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
-        notify:
-          - github_commit_status:
-              context: "Filebeat/Extended: Win-2019 Unit Tests"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
-          disk_size: 200
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "filebeat/build/*.xml"
-          - "filebeat/build/*.json"
+#  - group: "Filebeat Mandatory Testing"
+#    key: "mandatory-tests"
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+#
+#    steps:
+#      - label: ":ubuntu: Unit Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Unit Tests"
+#        agents:
+#          provider: gcp
+#          imageProject: elastic-images-qa
+#          image: family/platform-ingest-eng-prod-base-ubuntu-2204
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":ubuntu: Go Integration Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/integration-gotests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Integration Tests"
+#        agents:
+#          provider: gcp
+#          imageProject: elastic-images-qa
+#          image: family/platform-ingest-eng-prod-base-ubuntu-2204
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":ubuntu: Python Integration Tests"
+#        command:
+#          - ".buildkite/filebeat/scripts/integration-pytests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Python Integration Tests"
+#        agents:
+#          provider: gcp
+#          imageProject: elastic-images-qa
+#          image: family/platform-ingest-eng-prod-base-ubuntu-2204
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#      - label: ":windows:-{{matrix.image}} Unit Tests"
+#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat: Unit Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "{{matrix.image}}"
+#          machine_type: "n2-standard-8"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#        matrix:
+#          setup:
+#            image:
+#              - "${IMAGE_WIN_2016}"
+#              - "${IMAGE_WIN_2022}"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
+#
+#  - group: "Extended Testing"
+#    key: "extended-tests"
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
+#
+#    steps:
+#      - label: ":linux: ARM64 Unit Tests"
+#        key: "arm-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: Unit Tests ARM"
+#        agents:
+#          provider: "aws"
+#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+#          instanceType: "t4g.large"
+#        artifact_paths: "filebeat/build/*.xml"
+#
+#      - label: ":mac: MacOS Unit Tests"
+#        key: "macos-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+#        command:
+#          - ".buildkite/filebeat/scripts/unit-tests.sh"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: MacOS Unit Tests"
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_13}"
+#        artifact_paths: "filebeat/build/*.xml"
+#
+#  - group: "Windows Extended Testing"
+#    key: "extended-tests-win"
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+#
+#    steps:
+#      - label: ":windows: Win 2019 Unit Tests"
+#        key: "win-extended-2019"
+#        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+#        notify:
+#          - github_commit_status:
+#              context: "Filebeat/Extended: Win-2019 Unit Tests"
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2019}"
+#          machine_type: "n2-standard-8"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#        artifact_paths:
+#          - "filebeat/build/*.xml"
+#          - "filebeat/build/*.json"
 
   - group: "Packaging"
     key: "packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
-      - "extended-tests"
-      - "extended-tests-win"
+#    depends_on:
+#      - "mandatory-tests"
 
     steps:
-      - label: Package pipeline
+      - label: "Packaging"
         commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload"
+        if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_LABELS") != "skip-ci"

.buildkite/filebeat/scripts/package-step.sh

@@ -25,8 +25,9 @@ if are_files_changed "$changeset"; then
           - github_commit_status:
               context: "Filebeat/Packaging: Linux X86"
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          provider: gcp
+          imageProject: elastic-images-qa
+          image: family/platform-ingest-eng-prod-base-ubuntu-2204
 
       - label: ":linux: Packaging Linux ARM"
         key: "package-linux-arm"

.buildkite/filebeat/scripts/package.sh

@@ -4,9 +4,7 @@ set -euo pipefail
 
 source .buildkite/env-scripts/linux-env.sh
 
-echo ":: Evaluate Filebeat Changes ::"
-
 echo ":: Start Packaging ::"
-cd filebeat
-umask 0022
-mage package
+#cd filebeat
+#umask 0022
+mage -d filebeat package