added image tag and push

.buildkite/env-scripts/env.sh

@@ -1,13 +1,38 @@
 #!/usr/bin/env bash
 
+source .buildkite/env-scripts/util.sh
+
+docs_changeset="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+packaging_changeset="^dev-tools/packaging/
+^.go-version"
+
+REPO="beats"
 SETUP_GVM_VERSION="v0.5.1"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
+TMP_FOLDER="tmp.${REPO}"
+
+# Env vars for Packaging stage
+DOCKER_REGISTRY="docker.elastic.co"
+SNAPSHOT=true
+VERSION=$(make get-version | tr -d '\n')
+ONLY_DOCS=$(changeset_applies "$docs_changeset")
+PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
 
 export SETUP_GVM_VERSION
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
+export PACKAGING_CHANGES
+export ONLY_DOCS
+export GO_MOD_CHANGES
+export DOCKER_REGISTRY
+export SNAPSHOT
+export VERSION
+export REPO
+export TMP_FOLDER

.buildkite/env-scripts/linux-env.sh

@@ -18,7 +18,10 @@ if [[ $PLATFORM_TYPE == "Linux" ]]; then
   sudo apt-get install -y python3-venv
 fi
 
-echo ":: Setting up environment ::"
-add_bin_path
-with_go
-with_mage
+# Remove this code once beats specific ARM agent is set up
+if [[ $HW_TYPE == "aarch64" ]]; then
+  echo ":: Setting up environment ::"
+  add_bin_path
+  with_go
+  with_mage
+fi

.buildkite/env-scripts/util.sh

@@ -89,3 +89,40 @@ are_files_changed() {
     return 1;
   fi
 }
+
+changeset_applies() {
+  local changeset=$1
+  if are_files_changed "$changeset"; then
+    echo true
+  else
+    echo false
+  fi
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}
+
+google_cloud_logout_active_account() {
+  local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
+  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
+    echo "Logging out from GCP for active account"
+    gcloud auth revoke $active_account > /dev/null 2>&1
+  else
+    echo "No active GCP accounts found."
+  fi
+  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
+    unset GOOGLE_APPLICATION_CREDENTIALS
+    cleanup
+  fi
+}
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}

.buildkite/filebeat/filebeat-pipeline.yml

@@ -1,12 +1,14 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-eng-prod-base-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  SETUP_MAGE_VERSION: "1.14.0"
+  ASDF_MAGE_VERSION: "1.14.0"
 
 steps:
   - group: "Filebeat Mandatory Testing"
@@ -21,9 +23,9 @@ steps:
           - github_commit_status:
               context: "Filebeat: Unit Tests"
         agents:
-          provider: "gcp"
+          provider: gcp
+          imageProject: elastic-images-qa
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -35,9 +37,9 @@ steps:
           - github_commit_status:
               context: "Filebeat: Integration Tests"
         agents:
-          provider: "gcp"
+          provider: gcp
+          imageProject: elastic-images-qa
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -49,9 +51,9 @@ steps:
           - github_commit_status:
               context: "Filebeat: Python Integration Tests"
         agents:
-          provider: "gcp"
+          provider: gcp
+          imageProject: elastic-images-qa
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "c2-standard-16"
         artifact_paths:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
@@ -132,10 +134,8 @@ steps:
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
-      - "extended-tests"
-      - "extended-tests-win"
+#    depends_on:
+#      - "mandatory-tests"
 
     steps:
       - label: Package pipeline

.buildkite/filebeat/scripts/package-step.sh

@@ -25,8 +25,9 @@ if are_files_changed "$changeset"; then
           - github_commit_status:
               context: "Filebeat/Packaging: Linux X86"
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          provider: gcp
+          imageProject: elastic-images-qa
+          image: family/platform-ingest-eng-prod-base-ubuntu-2204
 
       - label: ":linux: Packaging Linux ARM"
         key: "package-linux-arm"

.buildkite/filebeat/scripts/package.sh

@@ -4,9 +4,7 @@ set -euo pipefail
 
 source .buildkite/env-scripts/linux-env.sh
 
-echo ":: Evaluate Filebeat Changes ::"
-
 echo ":: Start Packaging ::"
-cd filebeat
-umask 0022
-mage package
+#cd filebeat
+#umask 0022
+mage -d filebeat package

.buildkite/hooks/pre-command

@@ -6,6 +6,11 @@ source .buildkite/env-scripts/env.sh
 source .buildkite/env-scripts/util.sh
 source .buildkite/env-scripts/win-env.sh
 
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
+#PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
+GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
+
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
@@ -15,4 +20,16 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+
+    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+
+    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+    export GITHUB_TOKEN_SECRET=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+  fi
 fi

.buildkite/hooks/pre-exit

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+unset_secrets
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" && "$BUILDKITE_STEP_KEY" == package* ]]; then
+    google_cloud_logout_active_account
+    docker logout "${DOCKER_REGISTRY}"
+fi
+
+# Ensure that any temporal files created during any step are removed
+cleanup