From 2df811d118910593c3f629db6b3c78191f1fdbdc Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 1 Feb 2024 11:33:38 +0200 Subject: [PATCH] added image tag and push --- .buildkite/env-scripts/env.sh | 21 ++ .buildkite/env-scripts/linux-env.sh | 11 +- .buildkite/env-scripts/util.sh | 37 +++ .buildkite/filebeat/filebeat-pipeline.yml | 250 ++++++++++---------- .buildkite/filebeat/scripts/package-step.sh | 5 +- .buildkite/filebeat/scripts/package.sh | 8 +- .buildkite/hooks/pre-command | 17 ++ .buildkite/hooks/pre-exit | 15 ++ 8 files changed, 228 insertions(+), 136 deletions(-) create mode 100644 .buildkite/hooks/pre-exit diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index d94d03aad53b..fe0b6bf65c2c 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -1,13 +1,34 @@ #!/usr/bin/env bash +source .buildkite/env-scripts/util.sh + +docs_changeset="^.*\.(asciidoc|md)$ +deploy/kubernetes/.*-kubernetes.yaml" +packaging_changeset="^dev-tools/packaging/ +^.go-version" + SETUP_GVM_VERSION="v0.5.1" WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" +# Env vars for Packaging stage +DOCKER_REGISTRY="docker.elastic.co" +SNAPSHOT=true +VERSION=$(make get-version | tr -d '\n') +ONLY_DOCS=$(changeset_applies "$docs_changeset") +PACKAGING_CHANGES=$(changeset_applies "$packaging_changeset") +GO_MOD_CHANGES=$(changeset_applies "^go.mod") + export SETUP_GVM_VERSION export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE +export PACKAGING_CHANGES +export ONLY_DOCS +export GO_MOD_CHANGES +export DOCKER_REGISTRY +export SNAPSHOT +export VERSION diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh index edaf1a3100c2..266df2cf30d3 100644 --- a/.buildkite/env-scripts/linux-env.sh +++ b/.buildkite/env-scripts/linux-env.sh @@ -18,7 +18,10 @@ if [[ $PLATFORM_TYPE == "Linux" ]]; then sudo apt-get install -y python3-venv fi -echo ":: Setting up environment ::" -add_bin_path -with_go -with_mage +# Remove this code once beats specific ARM agent is set up +if [[ $HW_TYPE == "aarch64" ]]; then + echo ":: Setting up environment ::" + add_bin_path + with_go + with_mage +fi diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 157a5aff37af..747ae76e1545 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -89,3 +89,40 @@ are_files_changed() { return 1; fi } + +changeset_applies() { + local changeset=$1 + if are_files_changed "$changeset"; then + echo true + else + echo false + fi +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} + +google_cloud_logout_active_account() { + local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null) + if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then + echo "Logging out from GCP for active account" + gcloud auth revoke $active_account > /dev/null 2>&1 + else + echo "No active GCP accounts found." + fi + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then + unset GOOGLE_APPLICATION_CREDENTIALS + cleanup + fi +} + +cleanup() { + echo "Deleting temporary files..." + rm -rf ${BIN}/${TMP_FOLDER}.* + echo "Done." +} diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 7e4ba9180681..8ce7a0192483 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -7,136 +7,136 @@ env: IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_13: "generic-13-ventura-x64" + SETUP_MAGE_VERSION: "1.14.0" + ASDF_MAGE_VERSION: "1.14.0" steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat: Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-gotests.sh" - notify: - - github_commit_status: - context: "Filebeat: Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-pytests.sh" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" - notify: - - github_commit_status: - context: "Filebeat: Unit Tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" - - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" - artifact_paths: "filebeat/build/*.xml" - - - label: ":mac: MacOS Unit Tests" - key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_13}" - artifact_paths: "filebeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ - - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" - notify: - - github_commit_status: - context: "Filebeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" +# - group: "Filebeat Mandatory Testing" +# key: "mandatory-tests" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" +# +# steps: +# - label: ":ubuntu: Unit Tests" +# command: +# - ".buildkite/filebeat/scripts/unit-tests.sh" +# notify: +# - github_commit_status: +# context: "Filebeat: Unit Tests" +# agents: +# provider: gcp +# imageProject: elastic-images-qa +# image: family/platform-ingest-eng-prod-base-ubuntu-2204 +# artifact_paths: +# - "filebeat/build/*.xml" +# - "filebeat/build/*.json" +# +# - label: ":ubuntu: Go Integration Tests" +# command: +# - ".buildkite/filebeat/scripts/integration-gotests.sh" +# notify: +# - github_commit_status: +# context: "Filebeat: Integration Tests" +# agents: +# provider: gcp +# imageProject: elastic-images-qa +# image: family/platform-ingest-eng-prod-base-ubuntu-2204 +# artifact_paths: +# - "filebeat/build/*.xml" +# - "filebeat/build/*.json" +# +# - label: ":ubuntu: Python Integration Tests" +# command: +# - ".buildkite/filebeat/scripts/integration-pytests.sh" +# notify: +# - github_commit_status: +# context: "Filebeat: Python Integration Tests" +# agents: +# provider: gcp +# imageProject: elastic-images-qa +# image: family/platform-ingest-eng-prod-base-ubuntu-2204 +# artifact_paths: +# - "filebeat/build/*.xml" +# - "filebeat/build/*.json" +# +# - label: ":windows:-{{matrix.image}} Unit Tests" +# command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" +# notify: +# - github_commit_status: +# context: "Filebeat: Unit Tests" +# agents: +# provider: "gcp" +# image: "{{matrix.image}}" +# machine_type: "n2-standard-8" +# disk_size: 200 +# disk_type: "pd-ssd" +# matrix: +# setup: +# image: +# - "${IMAGE_WIN_2016}" +# - "${IMAGE_WIN_2022}" +# artifact_paths: +# - "filebeat/build/*.xml" +# - "filebeat/build/*.json" +# +# - group: "Extended Testing" +# key: "extended-tests" +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" +# +# steps: +# - label: ":linux: ARM64 Unit Tests" +# key: "arm-extended" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ +# command: +# - ".buildkite/filebeat/scripts/unit-tests.sh" +# notify: +# - github_commit_status: +# context: "Filebeat/Extended: Unit Tests ARM" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "t4g.large" +# artifact_paths: "filebeat/build/*.xml" +# +# - label: ":mac: MacOS Unit Tests" +# key: "macos-extended" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# command: +# - ".buildkite/filebeat/scripts/unit-tests.sh" +# notify: +# - github_commit_status: +# context: "Filebeat/Extended: MacOS Unit Tests" +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_13}" +# artifact_paths: "filebeat/build/*.xml" +# +# - group: "Windows Extended Testing" +# key: "extended-tests-win" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ +# +# steps: +# - label: ":windows: Win 2019 Unit Tests" +# key: "win-extended-2019" +# command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" +# notify: +# - github_commit_status: +# context: "Filebeat/Extended: Win-2019 Unit Tests" +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2019}" +# machine_type: "n2-standard-8" +# disk_size: 200 +# disk_type: "pd-ssd" +# artifact_paths: +# - "filebeat/build/*.xml" +# - "filebeat/build/*.json" - group: "Packaging" key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" - - "extended-tests" - - "extended-tests-win" +# depends_on: +# - "mandatory-tests" steps: - - label: Package pipeline + - label: "Packaging" commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_LABELS") != "skip-ci" diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh index a4127c3cd1d6..bd9ca52c5b09 100755 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -25,8 +25,9 @@ if are_files_changed "$changeset"; then - github_commit_status: context: "Filebeat/Packaging: Linux X86" agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-eng-prod-base-ubuntu-2204 - label: ":linux: Packaging Linux ARM" key: "package-linux-arm" diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 2ae226eb739c..a3a1a02af938 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -4,9 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Evaluate Filebeat Changes ::" - echo ":: Start Packaging ::" -cd filebeat -umask 0022 -mage package +#cd filebeat +#umask 0022 +mage -d filebeat package diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0a1567e53cd5..371db08686f7 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,6 +6,11 @@ source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" +#PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" +GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ ${PLATFORM_TYPE} = MINGW* ]]; then @@ -15,4 +20,16 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ -z "${GOLANG_VERSION-""}" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") fi + + if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then + export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + + export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + export GITHUB_TOKEN_SECRET=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) + export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH}) + export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH}) + fi fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 000000000000..77591cffbfaa --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +.buildkite/env-scripts/util.sh + +unset_secrets + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" && "$BUILDKITE_STEP_KEY" == package* ]]; then + google_cloud_logout_active_account + docker logout "${DOCKER_REGISTRY}" +fi + +# Ensure that any temporal files created during any step are removed +cleanup