Set build type for scheduled builds based on doc repo changes

.buildkite/build_pipeline.yml

@@ -1,30 +1,34 @@
 steps:
-  - input: "Build parameters"
-    if: build.source == "ui"
-    fields:
-      - select: "Rebuild?"
-        key: "REBUILD"
-        default: ""
-        required: false
-        options:
-          - label: "no"
-            value: ""
-          - label: "yes"
-            value: "rebuild"
-        hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release."
-      - select: "How should broken links be handled?"
-        key: "BROKEN_LINKS"
-        default: ""
-        required: false
-        options:
-          - label: "Continue without warning"
-            value: "skiplinkcheck"
-          - label: "Continue, but log a warning"
-            value: "warnlinkcheck"
-          - label: "Fail the build"
-            value: ""
-        hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail."
-  - wait
+  # - input: "Build parameters"
+  #   if: build.source == "ui"
+  #   fields:
+  #     - select: "Rebuild?"
+  #       key: "REBUILD"
+  #       default: ""
+  #       required: false
+  #       options:
+  #         - label: "no"
+  #           value: ""
+  #         - label: "yes"
+  #           value: "rebuild"
+  #       hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release."
+  #     - select: "How should broken links be handled?"
+  #       key: "BROKEN_LINKS"
+  #       default: ""
+  #       required: false
+  #       options:
+  #         - label: "Continue without warning"
+  #           value: "skiplinkcheck"
+  #         - label: "Continue, but log a warning"
+  #           value: "warnlinkcheck"
+  #         - label: "Fail the build"
+  #           value: ""
+  #       hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail."
+  # - wait
+  - label: "Full rebuild or incremental build?"
+    # this has to be commented out for testing purposes
+    #if: build.source == "schedule"
+    command: ".buildkite/scripts/compare_commits.sh"
   - label: ":white_check_mark: Build docs"
     command: |
       export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)"
@@ -34,7 +38,7 @@ steps:
       provider: "gcp"
       image: family/docs-ubuntu-2204
       machineType: ${BUILD_MACHINE_TYPE}
-    concurrency_group: build-docs
+    concurrency_group: build-docs-${BUILDKITE_BRANCH}
     concurrency: 1
 notify:
   - email: "[email protected]"

.buildkite/hooks/pre-command

@@ -34,6 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-pr" ]];then
     fi
 elif [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build" ]];then
     export BUILD_MACHINE_TYPE="n2-highcpu-32"
+    export BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field=value secret/ci/elastic-docs/buildkite_token)
 elif [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-air-gapped" ]] && [[ "$BUILDKITE_STEP_KEY" == "publish-air-gapped-doc" ]]; then
     export DOCKER_USERNAME=$(retry 5 vault kv get -field=username secret/ci/elastic-docs/docker.elastic.co)
     export DOCKER_PASSWORD=$(retry 5 vault kv get -field=password secret/ci/elastic-docs/docker.elastic.co)

.buildkite/scripts/compare_commits.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+last_successful_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}&state=passed"
+LAST_SUCCESSFUL_COMMIT=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[0].commit')
+
+echo "Comparing the current docs build commit ${BUILDKITE_COMMIT} to the last successful build commit ${LAST_SUCCESSFUL_COMMIT}"
+if [[ "$BUILDKITE_COMMIT" == "$LAST_SUCCESSFUL_COMMIT" ]]; then
+  echo "The docs repo has not changed since the last build."
+  buildkite-agent meta-data set "REBUILD" "false"
+else
+  echo "The docs repo has changed since the last build."
+  buildkite-agent meta-data set "REBUILD" "rebuild"
+fi