Merge remote-tracking branch 'upstream/main' into feature/updatecli-s…

…impler

* upstream/main: (28 commits)
  chore: Update to elastic/beats@19499ae5ab94 (elastic#13240)
  [updatecli] Bump elastic stack version to 8.15.0-561c6014 (elastic#13241)
  buildkite: create chainguard images (elastic#13226)
  chore: Update to elastic/beats@d5bfebb8940d (elastic#13231)
  [updatecli] Bump elastic stack version to 8.15.0-04e5793a (elastic#13227)
  build and push Docker image based on Chainguard base image (elastic#13137)
  buildkite(dra): support list for feature branches (elastic#13139)
  [updatecli] Bump elastic stack version to 8.15.0-e7873a67 (elastic#13218)
  chore: Update to elastic/beats@2419622b1b33 (elastic#13220)
  build(deps): bump the otel group with 3 updates (elastic#13213)
  [updatecli] Bump elastic stack version to 8.15.0-21ea2af3 (elastic#13206)
  chore: Update to elastic/beats@4a670391a6e2 (elastic#13209)
  build(deps): bump github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger from 0.100.0 to 0.101.0 in the jaeger group (elastic#13214)
  build(deps): bump the otel group with 2 updates (elastic#13205)
  Depend on apm-data plugin (elastic#12066)
  chore: Update to elastic/beats@d8072923498c (elastic#13203)
  [updatecli] Bump elastic stack version to 8.15.0-9bce1e63 (elastic#13200)
  build(deps): bump github.com/elastic/elastic-agent-system-metrics from 0.10.1 to 0.10.2 (elastic#13195)
  chore: Update to elastic/beats@062cc9b191b5 (elastic#13190)
  [updatecli] Bump elastic stack version to 8.15.0-1a872dbc (elastic#13185)
  ...

.buildkite/package.yml

@@ -7,6 +7,7 @@ notify:
 env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  GENERATE_CHAINGUARD_IMAGES: true
 
 steps:
   - group: "Package"

.buildkite/scripts/dra.sh

@@ -5,7 +5,7 @@
 ##
 ##  It downloads the generated artifacts and run the DRA only if the branch is an active
 ##  branch, based on the Unified Release policy. Otherwise, it won't run the DRA but print
-##  some traces.
+##  some traces and fail unless it's a feature branch then it will list the DRA artifacts.
 ##
 
 set -eo pipefail
@@ -28,6 +28,10 @@ if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "true" ]]; then
   exit 0
 fi
 
+# by default it uses the buildkite branch
+DRA_BRANCH="$BUILDKITE_BRANCH"
+# by default it publishes the DRA artifacts, for such it uses the collect command.
+dra_command=collect
 BRANCHES_URL=https://storage.googleapis.com/artifacts-api/snapshots/branches.json
 curl -s "${BRANCHES_URL}" > active-branches.json
 if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
@@ -37,30 +41,51 @@ if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
   echo "VERSION=$VERSION"
   echo "Supported branches:"
   cat active-branches.json
-  buildkite-agent annotate "${BUILDKITE_BRANCH} is not supported yet. Look for the supported branches in ${BRANCHES_URL}" --style 'warning' --context 'ctx-warn'
-  exit 1
+  if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
+    buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
+    dra_command=list
+
+    # use a different branch since DRA does not support feature branches but main/release branches
+    # for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
+    # to know if the branch was branched out from main or the release branches.
+    MAJOR_MINOR=${VERSION%.*}
+    if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
+      DRA_BRANCH=main
+    else
+      if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
+        DRA_BRANCH="$MAJOR_MINOR"
+      else
+        buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
+        exit 0
+      fi
+    fi
+  else
+    buildkite-agent annotate "${BUILDKITE_BRANCH} is not supported yet. Look for the supported branches in ${BRANCHES_URL}" --style 'warning' --context 'ctx-warn'
+    exit 1
+  fi
 fi
 
 dra() {
   local workflow=$1
-  echo "--- Run release manager $workflow"
+  local command=$2
+  echo "--- Run release manager $workflow (DRA command: $command)"
   docker run --rm \
     --name release-manager \
     -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
     -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
     -e VAULT_SECRET_ID="${VAULT_SECRET}" \
     --mount type=bind,readonly=false,src=$(pwd),target=/artifacts \
     docker.elastic.co/infra/release-manager:latest \
-      cli collect \
+      cli "$command" \
       --project apm-server \
-      --branch $BUILDKITE_BRANCH \
+      --branch $DRA_BRANCH \
       --commit $BUILDKITE_COMMIT \
       --workflow $workflow \
       --artifact-set main \
       --version $VERSION
 }
 
-dra "snapshot"
-if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then
-  dra "staging"
+dra "snapshot" "$dra_command"
+if [[ "${DRA_BRANCH}" != "main" ]]; then
+  dra "staging" "$dra_command"
 fi

.github/workflows/ci.yml

@@ -87,3 +87,28 @@ jobs:
           go-version-file: go.mod
           cache: false
       - run: make package-snapshot
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  test-package-and-push:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
+    env:
+      GENERATE_CHAINGUARD_IMAGES: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: false
+      - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current
+        with:
+          registry: docker.elastic.co
+          secret: secret/observability-team/ci/docker-registry/prod
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+      - run: make package-snapshot
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - run: make publish-docker-images